SherbyNet - /page/wikibook/entry.mu

<!DOCTYPE html>

<title>Gradient boosting</title>

</head>

Gradient boosting

</h1>

</header>

/* start https://en.wikipedia.org/ */

.mw-parser-output .hlist dl,.mw-parser-output .hlist ol,.mw-parser-output .hlist ul{margin:0;padding:0}.mw-parser-output .hlist dd,.mw-parser-output .hlist dt,.mw-parser-output .hlist li{margin:0;display:inline}.mw-parser-output .hlist.inline,.mw-parser-output .hlist.inline dl,.mw-parser-output .hlist.inline ol,.mw-parser-output .hlist.inline ul,.mw-parser-output .hlist dl dl,.mw-parser-output .hlist dl ol,.mw-parser-output .hlist dl ul,.mw-parser-output .hlist ol dl,.mw-parser-output .hlist ol ol,.mw-parser-output .hlist ol ul,.mw-parser-output .hlist ul dl,.mw-parser-output .hlist ul ol,.mw-parser-output .hlist ul ul{display:inline}.mw-parser-output .hlist .mw-empty-li{display:none}.mw-parser-output .hlist dt::after{content:": "}.mw-parser-output .hlist dd::after,.mw-parser-output .hlist li::after{content:" · ";font-weight:bold}.mw-parser-output .hlist dd:last-child::after,.mw-parser-output .hlist dt:last-child::after,.mw-parser-output .hlist li:last-child::after{content:none}.mw-parser-output .hlist dd dd:first-child::before,.mw-parser-output .hlist dd dt:first-child::before,.mw-parser-output .hlist dd li:first-child::before,.mw-parser-output .hlist dt dd:first-child::before,.mw-parser-output .hlist dt dt:first-child::before,.mw-parser-output .hlist dt li:first-child::before,.mw-parser-output .hlist li dd:first-child::before,.mw-parser-output .hlist li dt:first-child::before,.mw-parser-output .hlist li li:first-child::before{content:" (";font-weight:normal}.mw-parser-output .hlist dd dd:last-child::after,.mw-parser-output .hlist dd dt:last-child::after,.mw-parser-output .hlist dd li:last-child::after,.mw-parser-output .hlist dt dd:last-child::after,.mw-parser-output .hlist dt dt:last-child::after,.mw-parser-output .hlist dt li:last-child::after,.mw-parser-output .hlist li dd:last-child::after,.mw-parser-output .hlist li dt:last-child::after,.mw-parser-output .hlist li li:last-child::after{content:")";font-weight:normal}.mw-parser-output .hlist ol{counter-reset:listitem}.mw-parser-output .hlist ol>li{counter-increment:listitem}.mw-parser-output .hlist ol>li::before{content:" "counter(listitem)"\a0 "}.mw-parser-output .hlist dd ol>li:first-child::before,.mw-parser-output .hlist dt ol>li:first-child::before,.mw-parser-output .hlist li ol>li:first-child::before{content:" ("counter(listitem)"\a0 "}

/* end https://en.wikipedia.org/ */

</style><style data-mw-deduplicate="TemplateStyles:r1246091330">

/* start https://en.wikipedia.org/ */

.mw-parser-output .sidebar{width:22em;float:right;clear:right;margin:0.5em 0 1em 1em;background:var(--background-color-neutral-subtle,#f8f9fa);border:1px solid var(--border-color-base,#a2a9b1);padding:0.2em;text-align:center;line-height:1.4em;font-size:88%;border-collapse:collapse;display:table}body.skin-minerva .mw-parser-output .sidebar{display:table!important;float:right!important;margin:0.5em 0 1em 1em!important}.mw-parser-output .sidebar-subgroup{width:100%;margin:0;border-spacing:0}.mw-parser-output .sidebar-left{float:left;clear:left;margin:0.5em 1em 1em 0}.mw-parser-output .sidebar-none{float:none;clear:both;margin:0.5em 1em 1em 0}.mw-parser-output .sidebar-outer-title{padding:0 0.4em 0.2em;font-size:125%;line-height:1.2em;font-weight:bold}.mw-parser-output .sidebar-top-image{padding:0.4em}.mw-parser-output .sidebar-top-caption,.mw-parser-output .sidebar-pretitle-with-top-image,.mw-parser-output .sidebar-caption{padding:0.2em 0.4em 0;line-height:1.2em}.mw-parser-output .sidebar-pretitle{padding:0.4em 0.4em 0;line-height:1.2em}.mw-parser-output .sidebar-title,.mw-parser-output .sidebar-title-with-pretitle{padding:0.2em 0.8em;font-size:145%;line-height:1.2em}.mw-parser-output .sidebar-title-with-pretitle{padding:0.1em 0.4em}.mw-parser-output .sidebar-image{padding:0.2em 0.4em 0.4em}.mw-parser-output .sidebar-heading{padding:0.1em 0.4em}.mw-parser-output .sidebar-content{padding:0 0.5em 0.4em}.mw-parser-output .sidebar-content-with-subgroup{padding:0.1em 0.4em 0.2em}.mw-parser-output .sidebar-above,.mw-parser-output .sidebar-below{padding:0.3em 0.8em;font-weight:bold}.mw-parser-output .sidebar-collapse .sidebar-above,.mw-parser-output .sidebar-collapse .sidebar-below{border-top:1px solid #aaa;border-bottom:1px solid #aaa}.mw-parser-output .sidebar-navbar{text-align:right;font-size:115%;padding:0 0.4em 0.4em}.mw-parser-output .sidebar-list-title{padding:0 0.4em;text-align:left;font-weight:bold;line-height:1.6em;font-size:105%}.mw-parser-output .sidebar-list-title-c{padding:0 0.4em;text-align:center;margin:0 3.3em}@media(max-width:640px){body.mediawiki .mw-parser-output .sidebar{width:100%!important;clear:both;float:none!important;margin-left:0!important;margin-right:0!important}}body.skin--responsive .mw-parser-output .sidebar a>img{max-width:none!important}@media screen{html.skin-theme-clientpref-night .mw-parser-output .sidebar:not(.notheme) .sidebar-list-title,html.skin-theme-clientpref-night .mw-parser-output .sidebar:not(.notheme) .sidebar-title-with-pretitle{background:transparent!important}html.skin-theme-clientpref-night .mw-parser-output .sidebar:not(.notheme) .sidebar-title-with-pretitle a{color:var(--color-progressive)!important}}@media screen and (prefers-color-scheme:dark){html.skin-theme-clientpref-os .mw-parser-output .sidebar:not(.notheme) .sidebar-list-title,html.skin-theme-clientpref-os .mw-parser-output .sidebar:not(.notheme) .sidebar-title-with-pretitle{background:transparent!important}html.skin-theme-clientpref-os .mw-parser-output .sidebar:not(.notheme) .sidebar-title-with-pretitle a{color:var(--color-progressive)!important}}@media print{body.ns-0 .mw-parser-output .sidebar{display:none!important}}

/* end https://en.wikipedia.org/ */

</style><style data-mw-deduplicate="TemplateStyles:r886047488">

/* start https://en.wikipedia.org/ */

.mw-parser-output .nobold{font-weight:normal}

/* end https://en.wikipedia.org/ */

</style><table class="sidebar sidebar-collapse nomobile nowraplinks"><tbody><tr><td class="sidebar-pretitle">Part of a series on</td></tr><tr><th class="sidebar-title-with-pretitle"><a href="Machine_learning" title="Machine learning">Machine learning</a> and <a href="Data_mining" title="Data mining">data mining</a></th></tr><tr><td class="sidebar-content">

<div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)">Paradigms</div><div class="sidebar-list-content mw-collapsible-content hlist">

<ul><li><a href="Supervised_learning" title="Supervised learning">Supervised learning</a></li>

<li><a href="Unsupervised_learning" title="Unsupervised learning">Unsupervised learning</a></li>

<li><a href="Semi-supervised_learning" class="mw-redirect" title="Semi-supervised learning">Semi-supervised learning</a></li>

<li><a href="Self-supervised_learning" title="Self-supervised learning">Self-supervised learning</a></li>

<li><a href="Reinforcement_learning" title="Reinforcement learning">Reinforcement learning</a></li>

<li><a href="Meta-learning_(computer_science)" title="Meta-learning (computer science)">Meta-learning</a></li>

<li><a href="Online_machine_learning" title="Online machine learning">Online learning</a></li>

<li><a href="Batch_learning" class="mw-redirect" title="Batch learning">Batch learning</a></li>

<li><a href="Curriculum_learning" title="Curriculum learning">Curriculum learning</a></li>

<li><a href="Rule-based_machine_learning" title="Rule-based machine learning">Rule-based learning</a></li>

<li><a href="Neuro-symbolic_AI" title="Neuro-symbolic AI">Neuro-symbolic AI</a></li>

<li><a href="Neuromorphic_engineering" class="mw-redirect" title="Neuromorphic engineering">Neuromorphic engineering</a></li>

<li><a href="Quantum_machine_learning" title="Quantum machine learning">Quantum machine learning</a></li></ul></div></div></td>

</tr><tr><td class="sidebar-content">

<div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)">Problems</div><div class="sidebar-list-content mw-collapsible-content hlist">

<ul><li><a href="Statistical_classification" title="Statistical classification">Classification</a></li>

<li><a href="Generative_model" title="Generative model">Generative modeling</a></li>

<li><a href="Regression_analysis" title="Regression analysis">Regression</a></li>

<li><a href="Cluster_analysis" title="Cluster analysis">Clustering</a></li>

<li><a href="Dimensionality_reduction" title="Dimensionality reduction">Dimensionality reduction</a></li>

<li><a href="Density_estimation" title="Density estimation">Density estimation</a></li>

<li><a href="Anomaly_detection" title="Anomaly detection">Anomaly detection</a></li>

<li><a href="Data_cleaning" class="mw-redirect" title="Data cleaning">Data cleaning</a></li>

<li><a href="Automated_machine_learning" title="Automated machine learning">AutoML</a></li>

<li><a href="Association_rule_learning" title="Association rule learning">Association rules</a></li>

<li><a href="Semantic_analysis_(machine_learning)" title="Semantic analysis (machine learning)">Semantic analysis</a></li>

<li><a href="Structured_prediction" title="Structured prediction">Structured prediction</a></li>

<li><a href="Feature_engineering" title="Feature engineering">Feature engineering</a></li>

<li><a href="Feature_learning" title="Feature learning">Feature learning</a></li>

<li><a href="Learning_to_rank" title="Learning to rank">Learning to rank</a></li>

<li><a href="Grammar_induction" title="Grammar induction">Grammar induction</a></li>

<li><a href="Ontology_learning" title="Ontology learning">Ontology learning</a></li>

<li><a href="Multimodal_learning" title="Multimodal learning">Multimodal learning</a></li></ul></div></div></td>

</tr><tr><td class="sidebar-content">

<div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)"><div style="display: inline-block; line-height: 1.2em; padding: .1em 0;"><a href="Supervised_learning" title="Supervised learning">Supervised learning</a> (<a href="Statistical_classification" title="Statistical classification">classification</a> • <a href="Regression_analysis" title="Regression analysis">regression</a>) </div></div><div class="sidebar-list-content mw-collapsible-content hlist">

<ul><li><a href="Apprenticeship_learning" title="Apprenticeship learning">Apprenticeship learning</a></li>

<li><a href="Decision_tree_learning" title="Decision tree learning">Decision trees</a></li>

<li><a href="Ensemble_learning" title="Ensemble learning">Ensembles</a>

<ul><li><a href="Bootstrap_aggregating" title="Bootstrap aggregating">Bagging</a></li>

<li><a href="Boosting_(machine_learning)" title="Boosting (machine learning)">Boosting</a></li>

<li><a href="Random_forest" title="Random forest">Random forest</a></li></ul></li>

<li><a href="Linear_regression" title="Linear regression">Linear regression</a></li>

<li><a href="Naive_Bayes_classifier" title="Naive Bayes classifier">Naive Bayes</a></li>

<li><a href="Artificial_neural_network" class="mw-redirect" title="Artificial neural network">Artificial neural networks</a></li>

<li><a href="Logistic_regression" title="Logistic regression">Logistic regression</a></li>

<li><a href="Perceptron" title="Perceptron">Perceptron</a></li>

<li><a href="Relevance_vector_machine" title="Relevance vector machine">Relevance vector machine (RVM)</a></li>

<li><a href="Support_vector_machine" title="Support vector machine">Support vector machine (SVM)</a></li></ul></div></div></td>

</tr><tr><td class="sidebar-content">

<div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)"><a href="Cluster_analysis" title="Cluster analysis">Clustering</a></div><div class="sidebar-list-content mw-collapsible-content hlist">

<ul><li><a href="BIRCH" title="BIRCH">BIRCH</a></li>

<li><a href="Hierarchical_clustering" title="Hierarchical clustering">Hierarchical</a></li>

<li><a href="K-means_clustering" title="K-means clustering">k-means</a></li>

<li><a href="Fuzzy_clustering" title="Fuzzy clustering">Fuzzy</a></li>

<li><a href="Expectation%E2%80%93maximization_algorithm" title="Expectation–maximization algorithm">Expectation–maximization (EM)</a></li>

<li> <a href="DBSCAN" title="DBSCAN">DBSCAN</a></li>

<li><a href="OPTICS_algorithm" title="OPTICS algorithm">OPTICS</a></li>

<li><a href="Mean_shift" title="Mean shift">Mean shift</a></li></ul></div></div></td>

</tr><tr><td class="sidebar-content">

<div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)"><a href="Dimensionality_reduction" title="Dimensionality reduction">Dimensionality reduction</a></div><div class="sidebar-list-content mw-collapsible-content hlist">

<ul><li><a href="Factor_analysis" title="Factor analysis">Factor analysis</a></li>

</tr><tr><td class="sidebar-content">

<div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)"><a href="Structured_prediction" title="Structured prediction">Structured prediction</a></div><div class="sidebar-list-content mw-collapsible-content hlist">

<ul><li><a href="Graphical_model" title="Graphical model">Graphical models</a>

<ul><li><a href="Bayesian_network" title="Bayesian network">Bayes net</a></li>

<li><a href="Conditional_random_field" title="Conditional random field">Conditional random field</a></li>

<li><a href="Hidden_Markov_model" title="Hidden Markov model">Hidden Markov</a></li></ul></li></ul></div></div></td>

</tr><tr><td class="sidebar-content">

<div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)"><a href="Anomaly_detection" title="Anomaly detection">Anomaly detection</a></div><div class="sidebar-list-content mw-collapsible-content hlist">

<ul><li><a href="Random_sample_consensus" title="Random sample consensus">RANSAC</a></li>

<li><a href="Local_outlier_factor" title="Local outlier factor">Local outlier factor</a></li>

<li><a href="Isolation_forest" title="Isolation forest">Isolation forest</a></li></ul></div></div></td>

</tr><tr><td class="sidebar-content">

<div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)"><a href="Neural_network_(machine_learning)" title="Neural network (machine learning)">Neural networks</a></div><div class="sidebar-list-content mw-collapsible-content hlist">

<ul><li><a href="Autoencoder" title="Autoencoder">Autoencoder</a></li>

<li><a href="Deep_learning" title="Deep learning">Deep learning</a></li>

<li><a href="Feedforward_neural_network" title="Feedforward neural network">Feedforward neural network</a></li>

<li><a href="Recurrent_neural_network" title="Recurrent neural network">Recurrent neural network</a>

<li><a href="Reservoir_computing" title="Reservoir computing">reservoir computing</a></li></ul></li>

<li><a href="Boltzmann_machine" title="Boltzmann machine">Boltzmann machine</a>

<ul><li><a href="Restricted_Boltzmann_machine" title="Restricted Boltzmann machine">Restricted</a></li></ul></li>

<li><a href="Diffusion_model" title="Diffusion model">Diffusion model</a></li>

<li><a href="Convolutional_neural_network" title="Convolutional neural network">Convolutional neural network</a>

<li><a href="LeNet" title="LeNet">LeNet</a></li>

<li><a href="AlexNet" title="AlexNet">AlexNet</a></li>

<li><a href="DeepDream" title="DeepDream">DeepDream</a></li></ul></li>

<li><a href="Neural_field" title="Neural field">Neural field</a>

<ul><li><a href="Neural_radiance_field" title="Neural radiance field">Neural radiance field</a></li>

<li><a href="Physics-informed_neural_networks" title="Physics-informed neural networks">Physics-informed neural networks</a></li></ul></li>

<li><a href="Transformer_(deep_learning_architecture)" title="Transformer (deep learning architecture)">Transformer</a>

<ul><li><a href="Vision_transformer" title="Vision transformer">Vision</a></li></ul></li>

<li><a href="Mamba_(deep_learning_architecture)" title="Mamba (deep learning architecture)">Mamba</a></li>

<li><a href="Spiking_neural_network" title="Spiking neural network">Spiking neural network</a></li>

<li><a href="Memtransistor" title="Memtransistor">Memtransistor</a></li>

<li><a href="Electrochemical_RAM" title="Electrochemical RAM">Electrochemical RAM</a> (ECRAM)</li></ul></div></div></td>

</tr><tr><td class="sidebar-content">

<div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)"><a href="Reinforcement_learning" title="Reinforcement learning">Reinforcement learning</a></div><div class="sidebar-list-content mw-collapsible-content hlist">

<ul><li><a href="Q-learning" title="Q-learning">Q-learning</a></li>

<li><a href="Policy_gradient_method" title="Policy gradient method">Policy gradient</a></li>

<li><a href="State%E2%80%93action%E2%80%93reward%E2%80%93state%E2%80%93action" title="State–action–reward–state–action">SARSA</a></li>

<li><a href="Temporal_difference_learning" title="Temporal difference learning">Temporal difference (TD)</a></li>

<li><a href="Multi-agent_reinforcement_learning" title="Multi-agent reinforcement learning">Multi-agent</a>

</tr><tr><td class="sidebar-content">

<div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)">Learning with humans</div><div class="sidebar-list-content mw-collapsible-content hlist">

<ul><li><a href="Active_learning_(machine_learning)" title="Active learning (machine learning)">Active learning</a></li>

<li><a href="Crowdsourcing" title="Crowdsourcing">Crowdsourcing</a></li>

<li><a href="Human-in-the-loop" title="Human-in-the-loop">Human-in-the-loop</a></li>

<li><a href="Mechanistic_interpretability" title="Mechanistic interpretability">Mechanistic interpretability</a></li>

</tr><tr><td class="sidebar-content">

<div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)">Model diagnostics</div><div class="sidebar-list-content mw-collapsible-content hlist">

<ul><li><a href="Coefficient_of_determination" title="Coefficient of determination">Coefficient of determination</a></li>

<li><a href="Confusion_matrix" title="Confusion matrix">Confusion matrix</a></li>

<li><a href="Learning_curve_(machine_learning)" title="Learning curve (machine learning)">Learning curve</a></li>

<li><a href="Receiver_operating_characteristic" title="Receiver operating characteristic">ROC curve</a></li></ul></div></div></td>

</tr><tr><td class="sidebar-content">

<div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)">Mathematical foundations</div><div class="sidebar-list-content mw-collapsible-content hlist">

<ul><li><a href="Kernel_machines" class="mw-redirect" title="Kernel machines">Kernel machines</a></li>

<li><a href="Bias%E2%80%93variance_tradeoff" title="Bias–variance tradeoff">Bias–variance tradeoff</a></li>

<li><a href="Computational_learning_theory" title="Computational learning theory">Computational learning theory</a></li>

<li><a href="Empirical_risk_minimization" title="Empirical risk minimization">Empirical risk minimization</a></li>

<li><a href="Occam_learning" title="Occam learning">Occam learning</a></li>

<li><a href="Probably_approximately_correct_learning" title="Probably approximately correct learning">PAC learning</a></li>

<li><a href="Statistical_learning_theory" title="Statistical learning theory">Statistical learning</a></li>

<li><a href="Vapnik%E2%80%93Chervonenkis_theory" title="Vapnik–Chervonenkis theory">VC theory</a></li>

<li><a href="Topological_deep_learning" title="Topological deep learning">Topological deep learning</a></li></ul></div></div></td>

</tr><tr><td class="sidebar-content">

<div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)">Journals and conferences</div><div class="sidebar-list-content mw-collapsible-content hlist">

<li><a href="Conference_on_Neural_Information_Processing_Systems" title="Conference on Neural Information Processing Systems">NeurIPS</a></li>

<li><a href="International_Joint_Conference_on_Artificial_Intelligence" title="International Joint Conference on Artificial Intelligence">IJCAI</a></li>

</tr><tr><td class="sidebar-content">

<div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)">Related articles</div><div class="sidebar-list-content mw-collapsible-content hlist">

<ul><li><a href="Glossary_of_artificial_intelligence" title="Glossary of artificial intelligence">Glossary of artificial intelligence</a></li>

<li><a href="List_of_datasets_for_machine-learning_research" title="List of datasets for machine-learning research">List of datasets for machine-learning research</a>

<ul><li><a href="List_of_datasets_in_computer_vision_and_image_processing" title="List of datasets in computer vision and image processing">List of datasets in computer vision and image processing</a></li></ul></li>

<li><a href="Outline_of_machine_learning" title="Outline of machine learning">Outline of machine learning</a></li></ul></div></div></td>

</tr><tr><td class="sidebar-navbar"><style data-mw-deduplicate="TemplateStyles:r1239400231">

/* start https://en.wikipedia.org/ */

.mw-parser-output .navbar{display:inline;font-size:88%;font-weight:normal}.mw-parser-output .navbar-collapse{float:left;text-align:left}.mw-parser-output .navbar-boxtext{word-spacing:0}.mw-parser-output .navbar ul{display:inline-block;white-space:nowrap;line-height:inherit}.mw-parser-output .navbar-brackets::before{margin-right:-0.125em;content:"[ "}.mw-parser-output .navbar-brackets::after{margin-left:-0.125em;content:" ]"}.mw-parser-output .navbar li{word-spacing:-0.125em}.mw-parser-output .navbar a>span,.mw-parser-output .navbar a>abbr{text-decoration:inherit}.mw-parser-output .navbar-mini abbr{font-variant:small-caps;border-bottom:none;text-decoration:none;cursor:inherit}.mw-parser-output .navbar-ct-full{font-size:114%;margin:0 7em}.mw-parser-output .navbar-ct-mini{font-size:114%;margin:0 4em}html.skin-theme-clientpref-night .mw-parser-output .navbar li a abbr{color:var(--color-base)!important}@media(prefers-color-scheme:dark){html.skin-theme-clientpref-os .mw-parser-output .navbar li a abbr{color:var(--color-base)!important}}@media print{.mw-parser-output .navbar{display:none!important}}

/* end https://en.wikipedia.org/ */

</style></td></tr></tbody></table>

Gradient boosting is a <a href="Machine_learning" title="Machine learning">machine learning</a> technique based on <a href="Boosting_(machine_learning)" title="Boosting (machine learning)">boosting</a> in a functional space, where the target is pseudo-residuals instead of <a href="Residuals_(statistics)" class="mw-redirect" title="Residuals (statistics)">residuals</a> as in traditional boosting. It gives a prediction model in the form of an <a href="Ensemble_learning" title="Ensemble learning">ensemble</a> of weak prediction models, i.e., models that make very few assumptions about the data, which are typically simple <a href="Decision_tree_learning" title="Decision tree learning">decision trees</a>.<a href="#cite_note-hastie-1">[1]</a><a href="#cite_note-Friedman1999b-2">[2]</a> When a decision tree is the weak learner, the resulting algorithm is called gradient-boosted trees; it usually outperforms <a href="Random_forest" title="Random forest">random forest</a>.<a href="#cite_note-hastie-1">[1]</a> As with other <a href="Boosting_(machine_learning)" title="Boosting (machine learning)">boosting</a> methods, a gradient-boosted trees model is built in stages, but it generalizes the other methods by allowing optimization of an arbitrary <a href="Differentiable_function" title="Differentiable function">differentiable</a> <a href="Loss_function" title="Loss function">loss function</a>.

<div class="mw-heading mw-heading2"><h2 id="History">History</h2></div>

The idea of gradient boosting originated in the observation by <a href="Leo_Breiman" title="Leo Breiman">Leo Breiman</a> that boosting can be interpreted as an optimization algorithm on a suitable cost function.<a href="#cite_note-Breiman1997-3">[3]</a> Explicit regression gradient boosting algorithms were subsequently developed, by <a href="Jerome_H._Friedman" title="Jerome H. Friedman">Jerome H. Friedman</a>,<a href="#cite_note-Friedman1999a-4">[4]</a><a href="#cite_note-Friedman1999b-2">[2]</a> (in 1999 and later in 2001) simultaneously with the more general functional gradient boosting perspective of Llew Mason, Jonathan Baxter, Peter Bartlett and Marcus Frean.<a href="#cite_note-MasonBaxterBartlettFrean1999a-5">[5]</a><a href="#cite_note-MasonBaxterBartlettFrean1999b-6">[6]</a>

The latter two papers introduced the view of boosting algorithms as iterative functional gradient descent algorithms. That is, algorithms that optimize a cost function over function space by iteratively choosing a function (weak hypothesis) that points in the negative gradient direction. This functional gradient view of boosting has led to the development of boosting algorithms in many areas of machine learning and statistics beyond regression and classification.

<div class="mw-heading mw-heading2"><h2 id="Informal_introduction">Informal introduction</h2></div>

(This section follows the exposition by Cheng Li.<a href="#cite_note-7">[7]</a>)

Like other boosting methods, gradient boosting combines weak "learners" into a single strong learner iteratively. It is easiest to explain in the <a href="Least_squares" title="Least squares">least-squares</a> <a href="Regression_analysis" title="Regression analysis">regression</a> setting, where the goal is to teach a model <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle F}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle F}</annotation>

</semantics>

</math><img src="./545fd099af8541605f7ee55f08225526be88ce57.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.741ex; height:2.176ex;" alt="{\displaystyle F}" loading="lazy"> to predict values of the form <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\hat {y}}=F(x)}">

<mover>

</mover>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle {\hat {y}}=F(x)}</annotation>

</semantics>

</math><img src="./ad1ceaa8141c3c194c685cac4d222e286d88e1e6.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:9.28ex; height:2.843ex;" alt="{\displaystyle {\hat {y}}=F(x)}" loading="lazy"> by minimizing the <a href="Mean_squared_error" title="Mean squared error">mean squared error</a> <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\tfrac {1}{n}}\sum _{i}({\hat {y}}_{i}-y_{i})^{2}}">

<mfrac>

</mfrac>

</mstyle>

</mrow>

</mrow>

</munder>

<msub>

<mover>

</mover>

</mrow>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msup>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle {\tfrac {1}{n}}\sum _{i}({\hat {y}}_{i}-y_{i})^{2}}</annotation>

</semantics>

</math><img src="./7181a8dc0f6ec91f9b05013ad31ed32dadc25872.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -3.005ex; width:15.309ex; height:5.509ex;" alt="{\displaystyle {\tfrac {1}{n}}\sum _{i}({\hat {y}}_{i}-y_{i})^{2}}" loading="lazy">, where <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle i}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle i}</annotation>

</semantics>

</math><img src="./add78d8608ad86e54951b8c8bd6c8d8416533d20.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:0.802ex; height:2.176ex;" alt="{\displaystyle i}" loading="lazy"> indexes over some training set of size <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle n}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle n}</annotation>

</semantics>

</math><img src="./a601995d55609f2d9f5e233e36fbe9ea26011b3b.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.395ex; height:1.676ex;" alt="{\displaystyle n}" loading="lazy"> of actual values of the output variable <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle y}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle y}</annotation>

</semantics>

</math><img src="./b8a6208ec717213d4317e666f1ae872e00620a0d.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:1.155ex; height:2.009ex;" alt="{\displaystyle y}" loading="lazy">:

<msub>

<mover>

</mover>

</mrow>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle {\hat {y}}_{i}=}</annotation>

</semantics>

</math><img src="./4a6ee4141365a1f4a42db0c10c52d0e58ce29278.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:4.555ex; height:2.676ex;" alt="{\displaystyle {\hat {y}}_{i}=}" loading="lazy"> the predicted value <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle F(x_{i})}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle F(x_{i})}</annotation>

</semantics>

</math><img src="./54290e388c0dd2693499b3e01585c9755079c16c.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:5.679ex; height:2.843ex;" alt="{\displaystyle F(x_{i})}" loading="lazy"></li>

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle y_{i}=}</annotation>

</semantics>

</math><img src="./04a967219b4187e946ef6b9c7b895b3b4fd37d1f.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:4.392ex; height:2.009ex;" alt="{\displaystyle y_{i}=}" loading="lazy"> the observed value</li>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle n=}</annotation>

</semantics>

</math><img src="./ce0251053187f28804265b4ee74b25b5d86e1bba.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:3.848ex; height:1.676ex;" alt="{\displaystyle n=}" loading="lazy"> the number of samples in <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle y}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle y}</annotation>

</semantics>

</math><img src="./b8a6208ec717213d4317e666f1ae872e00620a0d.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:1.155ex; height:2.009ex;" alt="{\displaystyle y}" loading="lazy"></li></ul>

If the algorithm has <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle M}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle M}</annotation>

</semantics>

</math><img src="./f82cade9898ced02fdd08712e5f0c0151758a0dd.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:2.442ex; height:2.176ex;" alt="{\displaystyle M}" loading="lazy"> stages, at each stage <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle m}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle m}</annotation>

</semantics>

</math><img src="./0a07d98bb302f3856cbabc47b2b9016692e3f7bc.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:2.04ex; height:1.676ex;" alt="{\displaystyle m}" loading="lazy"> (<math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle 1\leq m\leq M}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle 1\leq m\leq M}</annotation>

</semantics>

</math><img src="./180b134ce5f780b4dc77bedbcc7e37a0e7b39260.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.505ex; width:11.842ex; height:2.343ex;" alt="{\displaystyle 1\leq m\leq M}" loading="lazy">), suppose some imperfect model <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle F_{m}}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle F_{m}}</annotation>

</semantics>

</math><img src="./afc15d41d3176d0fb9b4474762c53d49add76fbf.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:3.17ex; height:2.509ex;" alt="{\displaystyle F_{m}}" loading="lazy"> (for low <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle m}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle m}</annotation>

</semantics>

</math><img src="./0a07d98bb302f3856cbabc47b2b9016692e3f7bc.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:2.04ex; height:1.676ex;" alt="{\displaystyle m}" loading="lazy">, this model may simply predict <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\hat {y}}_{i}}">

<msub>

<mover>

</mover>

</mrow>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle {\hat {y}}_{i}}</annotation>

</semantics>

</math><img src="./5058f005e41a46a3c8d75449e20bac02bfe1609e.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:2.102ex; height:2.676ex;" alt="{\displaystyle {\hat {y}}_{i}}" loading="lazy"> to be <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\bar {y}}}">

<mover>

</mover>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle {\bar {y}}}</annotation>

</semantics>

</math><img src="./6b298744237368f34e61ff7dc90b34016a7037af.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:1.302ex; height:2.343ex;" alt="{\displaystyle {\bar {y}}}" loading="lazy">, the mean of <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle y}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle y}</annotation>

</semantics>

</math><img src="./b8a6208ec717213d4317e666f1ae872e00620a0d.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:1.155ex; height:2.009ex;" alt="{\displaystyle y}" loading="lazy">). In order to improve <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle F_{m}}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle F_{m}}</annotation>

</semantics>

</math><img src="./afc15d41d3176d0fb9b4474762c53d49add76fbf.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:3.17ex; height:2.509ex;" alt="{\displaystyle F_{m}}" loading="lazy">, our algorithm should add some new estimator, <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle h_{m}(x)}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle h_{m}(x)}</annotation>

</semantics>

</math><img src="./df5d8d5ba1497f33f67eb1721a3bb7616b32ad8e.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:6.153ex; height:2.843ex;" alt="{\displaystyle h_{m}(x)}" loading="lazy">. Thus,

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle F_{m+1}(x_{i})=F_{m}(x_{i})+h_{m}(x_{i})=y_{i}}</annotation>

</semantics>

</math><img src="./274ce447eaefd2f6687a4ac444828a66d5a2e120.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:34.246ex; height:2.843ex;" alt="{\displaystyle F_{m+1}(x_{i})=F_{m}(x_{i})+h_{m}(x_{i})=y_{i}}" loading="lazy"></dd></dl>

or, equivalently,

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle h_{m}(x_{i})=y_{i}-F_{m}(x_{i})}</annotation>

</semantics>

</math><img src="./c0959be83fbfb9ec6f2189473400a2a236a8f2c1.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:21.939ex; height:2.843ex;" alt="{\displaystyle h_{m}(x_{i})=y_{i}-F_{m}(x_{i})}" loading="lazy">.</dd></dl>

Therefore, gradient boosting will fit <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle h_{m}}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle h_{m}}</annotation>

</semantics>

</math><img src="./790122b38658a2f398fe85b547a3f7f6db76d324.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:3.014ex; height:2.509ex;" alt="{\displaystyle h_{m}}" loading="lazy"> to the <a href="Errors_and_residuals" title="Errors and residuals">residual</a> <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle y_{i}-F_{m}(x_{i})}">

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle y_{i}-F_{m}(x_{i})}</annotation>

</semantics>

</math><img src="./627fdd06afec90633767eb85150c4f01fbe76ca2.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:11.888ex; height:2.843ex;" alt="{\displaystyle y_{i}-F_{m}(x_{i})}" loading="lazy">. As in other boosting variants, each <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle F_{m+1}}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle F_{m+1}}</annotation>

</semantics>

</math><img src="./f79d37287d73b20d8cab41477cda062c8b9e8912.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:5.27ex; height:2.509ex;" alt="{\displaystyle F_{m+1}}" loading="lazy"> attempts to correct the errors of its predecessor <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle F_{m}}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle F_{m}}</annotation>

</semantics>

</math><img src="./afc15d41d3176d0fb9b4474762c53d49add76fbf.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:3.17ex; height:2.509ex;" alt="{\displaystyle F_{m}}" loading="lazy">. A generalization of this idea to <a href="Loss_function" title="Loss function">loss functions</a> other than squared error, and to <a href="Learning_to_rank" title="Learning to rank">classification and ranking problems</a>, follows from the observation that residuals <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle h_{m}(x_{i})}">

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle h_{m}(x_{i})}</annotation>

</semantics>

</math><img src="./8cf62c65752e938197f5f618600e53a63b50f518.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:6.953ex; height:2.843ex;" alt="{\displaystyle h_{m}(x_{i})}" loading="lazy"> for a given model are proportional to the negative gradients of the <a href="Mean_squared_error" title="Mean squared error">mean squared error (MSE)</a> loss function (with respect to <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle F(x_{i})}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle F(x_{i})}</annotation>

</semantics>

</math><img src="./54290e388c0dd2693499b3e01585c9755079c16c.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:5.679ex; height:2.843ex;" alt="{\displaystyle F(x_{i})}" loading="lazy">):

<msub>

</mrow>

</msub>

<mfrac>

</mfrac>

</mrow>

</mrow>

</mrow>

</munderover>

<msup>

<mrow>

<mrow>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mrow>

</mrow>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle L_{\rm {MSE}}={\frac {1}{n}}\sum _{i=1}^{n}\left(y_{i}-F(x_{i})\right)^{2}}</annotation>

</semantics>

</math><img src="./d62c1b9c4ccc3f6f20394c797d28d79e4b0b4232.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -3.005ex; width:28.136ex; height:6.843ex;" alt="{\displaystyle L_{\rm {MSE}}={\frac {1}{n}}\sum _{i=1}^{n}\left(y_{i}-F(x_{i})\right)^{2}}" loading="lazy"></dd></dl>

<mfrac>

<mrow>

<msub>

</mrow>

</msub>

</mrow>

<mrow>

<msub>

</mrow>

</msub>

</mrow>

</mfrac>

</mrow>

<mfrac>

</mfrac>

</mrow>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<mfrac>

</mfrac>

</mrow>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle -{\frac {\partial L_{\rm {MSE}}}{\partial F(x_{i})}}={\frac {2}{n}}(y_{i}-F(x_{i}))={\frac {2}{n}}h_{m}(x_{i})}</annotation>

</semantics>

</math><img src="./4597c37aa2ad47989f5b1842545afa69ec725f40.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -2.671ex; width:39.521ex; height:6.176ex;" alt="{\displaystyle -{\frac {\partial L_{\rm {MSE}}}{\partial F(x_{i})}}={\frac {2}{n}}(y_{i}-F(x_{i}))={\frac {2}{n}}h_{m}(x_{i})}" loading="lazy">.</dd></dl>

So, gradient boosting could be generalized to a <a href="Gradient_descent" title="Gradient descent">gradient descent</a> algorithm by plugging in a different loss and its gradient.

<div class="mw-heading mw-heading2"><h2 id="Algorithm">Algorithm</h2></div>

Many <a href="Supervised_learning" title="Supervised learning">supervised learning</a> problems involve an output variable y and a vector of input variables x, related to each other with some probabilistic distribution. The goal is to find some function <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\hat {F}}(x)}">

<mover>

</mover>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle {\hat {F}}(x)}</annotation>

</semantics>

</math><img src="./9dd32eef5572ac89f3a31f9f0c54a2c7b070d90c.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:4.944ex; height:3.343ex;" alt="{\displaystyle {\hat {F}}(x)}" loading="lazy"> that best approximates the output variable from the values of input variables. This is formalized by introducing some <a href="Loss_function" title="Loss function">loss function</a> <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle L(y,F(x))}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle L(y,F(x))}</annotation>

</semantics>

</math><img src="./7da73aa7583b8fa4bb3729fbe01cbf42a5eacb8d.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:10.461ex; height:2.843ex;" alt="{\displaystyle L(y,F(x))}" loading="lazy"> and minimizing it in expectation:

<mover>

</mover>

</mrow>

<mrow>

</mrow>

</munder>

</mrow>

<msub>

</mrow>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle {\hat {F}}={\underset {F}{\arg \min }}\,\mathbb {E} _{x,y}[L(y,F(x))]}</annotation>

</semantics>

</math><img src="./655597bea9c34023b858924a1cf23b177042291c.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -2.505ex; width:28.542ex; height:5.009ex;" alt="{\displaystyle {\hat {F}}={\underset {F}{\arg \min }}\,\mathbb {E} _{x,y}[L(y,F(x))]}" loading="lazy">.</dd></dl>

The gradient boosting method assumes a real-valued y. It seeks an approximation <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\hat {F}}(x)}">

<mover>

</mover>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle {\hat {F}}(x)}</annotation>

</semantics>

</math><img src="./9dd32eef5572ac89f3a31f9f0c54a2c7b070d90c.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:4.944ex; height:3.343ex;" alt="{\displaystyle {\hat {F}}(x)}" loading="lazy"> in the form of a weighted sum of M functions <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle h_{m}(x)}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle h_{m}(x)}</annotation>

</semantics>

</math><img src="./df5d8d5ba1497f33f67eb1721a3bb7616b32ad8e.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:6.153ex; height:2.843ex;" alt="{\displaystyle h_{m}(x)}" loading="lazy"> from some class <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\mathcal {H}}}">

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle {\mathcal {H}}}</annotation>

</semantics>

</math><img src="./19ef4c7b923a5125ac91aa491838a95ee15b804f.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.964ex; height:2.176ex;" alt="{\displaystyle {\mathcal {H}}}" loading="lazy">, called base (or <a href="Weak_learner" class="mw-redirect" title="Weak learner">weak</a>) learners:

<mover>

</mover>

</mrow>

</mrow>

</mrow>

</munderover>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<mtext>const</mtext>

</mstyle>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle {\hat {F}}(x)=\sum _{m=1}^{M}\gamma _{m}h_{m}(x)+{\mbox{const}}}</annotation>

</semantics>

</math><img src="./70313f8594de6a89e88a64eb6a5ffeb125b1cb11.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -3.005ex; width:29.154ex; height:7.343ex;" alt="{\displaystyle {\hat {F}}(x)=\sum _{m=1}^{M}\gamma _{m}h_{m}(x)+{\mbox{const}}}" loading="lazy">,</dd></dl>

where <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \gamma _{m}}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \gamma _{m}}</annotation>

</semantics>

</math><img src="./27a890b6f03d91a6f3f1033e4ffdf6b2c46c7737.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:2.879ex; height:2.176ex;" alt="{\displaystyle \gamma _{m}}" loading="lazy"> is the weight at stage <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle m}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle m}</annotation>

</semantics>

</math><img src="./0a07d98bb302f3856cbabc47b2b9016692e3f7bc.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:2.04ex; height:1.676ex;" alt="{\displaystyle m}" loading="lazy">. We are usually given a training set <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \{(x_{1},y_{1}),\dots ,(x_{n},y_{n})\}}">

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \{(x_{1},y_{1}),\dots ,(x_{n},y_{n})\}}</annotation>

</semantics>

</math><img src="./1805d7245b2f223e2323f78f5e9f353e38f8cffb.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:22.673ex; height:2.843ex;" alt="{\displaystyle \{(x_{1},y_{1}),\dots ,(x_{n},y_{n})\}}" loading="lazy"> of known values of x and corresponding values of y. In accordance with the <a href="Empirical_risk_minimization" title="Empirical risk minimization">empirical risk minimization</a> principle, the method tries to find an approximation <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\hat {F}}(x)}">

<mover>

</mover>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle {\hat {F}}(x)}</annotation>

</semantics>

</math><img src="./9dd32eef5572ac89f3a31f9f0c54a2c7b070d90c.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:4.944ex; height:3.343ex;" alt="{\displaystyle {\hat {F}}(x)}" loading="lazy"> that minimizes the average value of the loss function on the training set, i.e., minimizes the empirical risk. It does so by starting with a model, consisting of a constant function <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle F_{0}(x)}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle F_{0}(x)}</annotation>

</semantics>

</math><img src="./fa7b801804c1fd3f8e27dbfa5eaf7ae94e806fd0.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:5.688ex; height:2.843ex;" alt="{\displaystyle F_{0}(x)}" loading="lazy">, and incrementally expands it in a <a href="Greedy_algorithm" title="Greedy algorithm">greedy</a> fashion:

<msub>

</mrow>

</msub>

<mrow>

</mrow>

<mrow>

<msub>

</mrow>

</msub>

</mrow>

</munder>

</mrow>

</mrow>

</mrow>

</munderover>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle F_{0}(x)={\underset {h_{m}\in {\mathcal {H}}}{\arg \min }}{\sum _{i=1}^{n}{L(y_{i},h_{m}(x_{i}))}}}</annotation>

</semantics>

</math><img src="./5f0d12ea6558e55c612c70b5649944848c86bff7.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -3.005ex; width:33.345ex; height:6.843ex;" alt="{\displaystyle F_{0}(x)={\underset {h_{m}\in {\mathcal {H}}}{\arg \min }}{\sum _{i=1}^{n}{L(y_{i},h_{m}(x_{i}))}}}" loading="lazy">,</dd>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<mrow>

<mrow>

</mrow>

<mrow>

<msub>

</mrow>

</msub>

</mrow>

</munder>

</mrow>

<mrow>

</mrow>

</mrow>

</munderover>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mrow>

</mrow>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle F_{m}(x)=F_{m-1}(x)+\left({\underset {h_{m}\in {\mathcal {H}}}{\operatorname {arg\,min} }}\left[{\sum _{i=1}^{n}{L(y_{i},F_{m-1}(x_{i})+h_{m}(x_{i}))}}\right]\right)(x)}</annotation>

</semantics>

</math><img src="./4c896da6f2650ec46b24d77b3f1ca2d2ff7b4a1c.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -3.171ex; width:67.569ex; height:7.509ex;" alt="{\displaystyle F_{m}(x)=F_{m-1}(x)+\left({\underset {h_{m}\in {\mathcal {H}}}{\operatorname {arg\,min} }}\left[{\sum _{i=1}^{n}{L(y_{i},F_{m-1}(x_{i})+h_{m}(x_{i}))}}\right]\right)(x)}" loading="lazy">,</dd></dl>

for <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle m\geq 1}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle m\geq 1}</annotation>

</semantics>

</math><img src="./1e0f3243e9d7f06bee548558bf20aaa9b5263d3f.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.505ex; width:6.301ex; height:2.343ex;" alt="{\displaystyle m\geq 1}" loading="lazy">, where <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle h_{m}\in {\mathcal {H}}}">

<msub>

</mrow>

</msub>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle h_{m}\in {\mathcal {H}}}</annotation>

</semantics>

</math><img src="./df97ad2e47ca788a1bc11b748753bb805761e57c.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:7.818ex; height:2.509ex;" alt="{\displaystyle h_{m}\in {\mathcal {H}}}" loading="lazy"> is a base learner function.

Unfortunately, choosing the best function <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle h_{m}}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle h_{m}}</annotation>

</semantics>

</math><img src="./790122b38658a2f398fe85b547a3f7f6db76d324.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:3.014ex; height:2.509ex;" alt="{\displaystyle h_{m}}" loading="lazy"> at each step for an arbitrary loss function L is a computationally infeasible optimization problem in general. Therefore, we restrict our approach to a simplified version of the problem. The idea is to apply a <a href="Steepest_descent" class="mw-redirect" title="Steepest descent">steepest descent</a> step to this minimization problem (functional gradient descent). The basic idea is to find a local minimum of the loss function by iterating on <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle F_{m-1}(x)}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle F_{m-1}(x)}</annotation>

</semantics>

</math><img src="./f66f1c4a6635e79b33f6a7b2a28d702a11548857.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:8.409ex; height:2.843ex;" alt="{\displaystyle F_{m-1}(x)}" loading="lazy">. In fact, the local maximum-descent direction of the loss function is the negative gradient.<a href="#cite_note-8">[8]</a> Hence, moving a small amount <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \gamma }">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \gamma }</annotation>

</semantics>

</math><img src="./a223c880b0ce3da8f64ee33c4f0010beee400b1a.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:1.262ex; height:2.176ex;" alt="{\displaystyle \gamma }" loading="lazy"> such that the linear approximation remains valid:

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mrow>

</mrow>

</munderover>

<msub>

<msub>

</mrow>

</msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle F_{m}(x)=F_{m-1}(x)-\gamma \sum _{i=1}^{n}{\nabla _{F_{m-1}}L(y_{i},F_{m-1}(x_{i}))}}</annotation>

</semantics>

</math><img src="./2716c372d1ba9f07e4f47c9db81815ad9b122947.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -3.005ex; width:47.888ex; height:6.843ex;" alt="{\displaystyle F_{m}(x)=F_{m-1}(x)-\gamma \sum _{i=1}^{n}{\nabla _{F_{m-1}}L(y_{i},F_{m-1}(x_{i}))}}" loading="lazy">

where <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \gamma >0}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \gamma >0}</annotation>

</semantics>

</math><img src="./775a6435e4270cddf2cd7dcb486c20f7f4bb8cee.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:5.523ex; height:2.676ex;" alt="{\displaystyle \gamma >0}" loading="lazy">. For small <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \gamma }">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \gamma }</annotation>

</semantics>

</math><img src="./a223c880b0ce3da8f64ee33c4f0010beee400b1a.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:1.262ex; height:2.176ex;" alt="{\displaystyle \gamma }" loading="lazy">, this implies that <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle L(y_{i},F_{m}(x_{i}))\leq L(y_{i},F_{m-1}(x_{i}))}">

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle L(y_{i},F_{m}(x_{i}))\leq L(y_{i},F_{m-1}(x_{i}))}</annotation>

</semantics>

</math><img src="./61f1bff410e687f9ba7ffc207b39bc5690022cc3.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:32.146ex; height:2.843ex;" alt="{\displaystyle L(y_{i},F_{m}(x_{i}))\leq L(y_{i},F_{m-1}(x_{i}))}" loading="lazy">.

/* start https://en.wikipedia.org/ */

.mw-parser-output .cot-header-mainspace{background:#F0F2F5;color:inherit}.mw-parser-output .cot-header-other{background:#CCFFCC;color:inherit}@media screen{html.skin-theme-clientpref-night .mw-parser-output .cot-header-mainspace{background:#14181F;color:inherit}html.skin-theme-clientpref-night .mw-parser-output .cot-header-other{background:#003500;color:inherit}}@media screen and (prefers-color-scheme:dark){html.skin-theme-clientpref-os .mw-parser-output .cot-header-mainspace{background:#14181F;color:inherit}html.skin-theme-clientpref-os .mw-parser-output .cot-header-other{background:#003500;color:inherit}}

/* end https://en.wikipedia.org/ */

</style>

<th class="cot-header-mainspace" style="; font-size:87%; padding:0.2em 0.3em; text-align:center;"><div style="font-size:115%;margin:0 4em">Proof of functional form of derivative</div>

</th></tr>

<tr>

<td style="color:inherit; border: solid 1px Silver; padding: 0.6em; background: var(--background-color-base, #fff);">To prove the following, consider the objective

</mrow>

</mrow>

</munderover>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle O=\sum _{i=1}^{n}{L(y_{i},F_{m-1}(x_{i})+h_{m}(x_{i}))}}</annotation>

</semantics>

</math><img src="./0a40edaa93f901e2e325aef11c143c440dc79bf4.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -3.005ex; width:33.981ex; height:6.843ex;" alt="{\displaystyle O=\sum _{i=1}^{n}{L(y_{i},F_{m-1}(x_{i})+h_{m}(x_{i}))}}" loading="lazy">

Doing a Taylor expansion around the fixed point <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle F_{m-1}(x_{i})}">

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle F_{m-1}(x_{i})}</annotation>

</semantics>

</math><img src="./863a341d311c7c86471d76c02e5d5c6fffab87b5.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:9.209ex; height:2.843ex;" alt="{\displaystyle F_{m-1}(x_{i})}" loading="lazy"> up to first order

</mrow>

</mrow>

</munderover>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mrow>

</mrow>

</mrow>

</munderover>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

<msub>

</mrow>

</msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle O=\sum _{i=1}^{n}{L(y_{i},F_{m-1}(x_{i})+h_{m}(x_{i}))}\approx \sum _{i=1}^{n}{L(y_{i},F_{m-1}(x_{i}))+h_{m}(x_{i})\nabla _{F_{m-1}}L(y_{i},F_{m-1}(x_{i}))}+\ldots }</annotation>

</semantics>

</math><img src="./ee2c2b51095f7943a9dec4208cd4559796e38daf.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -3.005ex; width:93.591ex; height:6.843ex;" alt="{\displaystyle O=\sum _{i=1}^{n}{L(y_{i},F_{m-1}(x_{i})+h_{m}(x_{i}))}\approx \sum _{i=1}^{n}{L(y_{i},F_{m-1}(x_{i}))+h_{m}(x_{i})\nabla _{F_{m-1}}L(y_{i},F_{m-1}(x_{i}))}+\ldots }" loading="lazy">

Now differentiating w.r.t to <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle h_{m}(x_{i})}">

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle h_{m}(x_{i})}</annotation>

</semantics>

</math><img src="./8cf62c65752e938197f5f618600e53a63b50f518.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:6.953ex; height:2.843ex;" alt="{\displaystyle h_{m}(x_{i})}" loading="lazy">, only the derivative of the second term remains

<msub>

<msub>

</mrow>

</msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \nabla _{F_{m-1}}L(y_{i},F_{m-1}(x_{i}))}</annotation>

</semantics>

</math><img src="./c36257b5ce32517d6ace00a99853dfe651fb1aab.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:21.84ex; height:3.009ex;" alt="{\displaystyle \nabla _{F_{m-1}}L(y_{i},F_{m-1}(x_{i}))}" loading="lazy">. This is the direction of steepest ascent and hence we must move in the opposite (i.e., negative) direction in order to move in the direction of steepest descent.

</td></tr></tbody></table></div>

Furthermore, we can optimize <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \gamma }">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \gamma }</annotation>

</semantics>

</math><img src="./a223c880b0ce3da8f64ee33c4f0010beee400b1a.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:1.262ex; height:2.176ex;" alt="{\displaystyle \gamma }" loading="lazy"> by finding the <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \gamma }">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \gamma }</annotation>

</semantics>

</math><img src="./a223c880b0ce3da8f64ee33c4f0010beee400b1a.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:1.262ex; height:2.176ex;" alt="{\displaystyle \gamma }" loading="lazy"> value for which the loss function has a minimum:

<msub>

</mrow>

</msub>

<mrow>

</mrow>

</munder>

</mrow>

</mrow>

</mrow>

</munderover>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mrow>

<mrow>

</mrow>

</munder>

</mrow>

</mrow>

</mrow>

</munderover>

<mrow>

<mrow>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

<msub>

</mrow>

</msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mrow>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \gamma _{m}={\underset {\gamma }{\arg \min }}{\sum _{i=1}^{n}{L(y_{i},F_{m}(x_{i}))}}={\underset {\gamma }{\arg \min }}{\sum _{i=1}^{n}{L\left(y_{i},F_{m-1}(x_{i})-\gamma \nabla _{F_{m-1}}L(y_{i},F_{m-1}(x_{i}))\right)}}.}</annotation>

</semantics>

</math><img src="./5ccde548118369573b2b895dbe5cf40e5bedc51b.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -3.005ex; width:87.902ex; height:6.843ex;" alt="{\displaystyle \gamma _{m}={\underset {\gamma }{\arg \min }}{\sum _{i=1}^{n}{L(y_{i},F_{m}(x_{i}))}}={\underset {\gamma }{\arg \min }}{\sum _{i=1}^{n}{L\left(y_{i},F_{m-1}(x_{i})-\gamma \nabla _{F_{m-1}}L(y_{i},F_{m-1}(x_{i}))\right)}}.}" loading="lazy">

If we considered the continuous case, i.e., where <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\mathcal {H}}}">

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle {\mathcal {H}}}</annotation>

</semantics>

</math><img src="./19ef4c7b923a5125ac91aa491838a95ee15b804f.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.964ex; height:2.176ex;" alt="{\displaystyle {\mathcal {H}}}" loading="lazy"> is the set of arbitrary differentiable functions on <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \mathbb {R} }">

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \mathbb {R} }</annotation>

</semantics>

</math><img src="./786849c765da7a84dbc3cce43e96aad58a5868dc.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.678ex; height:2.176ex;" alt="{\displaystyle \mathbb {R} }" loading="lazy">, we would update the model in accordance with the following equations

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mrow>

</mrow>

</munderover>

<msub>

<msub>

</mrow>

</msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle F_{m}(x)=F_{m-1}(x)-\gamma _{m}\sum _{i=1}^{n}{\nabla _{F_{m-1}}L(y_{i},F_{m-1}(x_{i}))}}</annotation>

</semantics>

</math><img src="./5cd5f02a9eeb356da80bfa0a99ba98a8b39f96ce.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -3.005ex; width:49.505ex; height:6.843ex;" alt="{\displaystyle F_{m}(x)=F_{m-1}(x)-\gamma _{m}\sum _{i=1}^{n}{\nabla _{F_{m-1}}L(y_{i},F_{m-1}(x_{i}))}}" loading="lazy"></dd></dl>

where <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \gamma _{m}}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \gamma _{m}}</annotation>

</semantics>

</math><img src="./27a890b6f03d91a6f3f1033e4ffdf6b2c46c7737.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:2.879ex; height:2.176ex;" alt="{\displaystyle \gamma _{m}}" loading="lazy"> is the step length, defined as

<msub>

</mrow>

</msub>

<mrow>

</mrow>

</munder>

</mrow>

</mrow>

</mrow>

</munderover>

<mrow>

<mrow>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

<msub>

</mrow>

</msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mrow>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \gamma _{m}={\underset {\gamma }{\arg \min }}{\sum _{i=1}^{n}{L\left(y_{i},F_{m-1}(x_{i})-\gamma \nabla _{F_{m-1}}L(y_{i},F_{m-1}(x_{i}))\right)}}.}</annotation>

</semantics>

</math>

In the discrete case however, i.e. when the set <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\mathcal {H}}}">

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle {\mathcal {H}}}</annotation>

</semantics>

</math><img src="./19ef4c7b923a5125ac91aa491838a95ee15b804f.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.964ex; height:2.176ex;" alt="{\displaystyle {\mathcal {H}}}" loading="lazy"> is finite, we choose the candidate function h closest to the gradient of L for which the coefficient γ may then be calculated with the aid of <a href="Line_search" title="Line search">line search</a> on the above equations. Note that this approach is a heuristic and therefore doesn't yield an exact solution to the given problem, but rather an approximation.

In pseudocode, the generic gradient boosting method is:<a href="#cite_note-Friedman1999a-4">[4]</a><a href="#cite_note-hastie-1">[1]</a>

/* start https://en.wikipedia.org/ */

.mw-parser-output .framebox-container{margin-bottom:1.25em}.mw-parser-output .framebox-header{height:8px;margin:0;border:0;padding:0;font-size:1px}.mw-parser-output .framebox-inner{padding:5px;font-size:85%}

/* end https://en.wikipedia.org/ */

</style>

Input: training set <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \{(x_{i},y_{i})\}_{i=1}^{n},}">

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mrow>

</mrow>

</msubsup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \{(x_{i},y_{i})\}_{i=1}^{n},}</annotation>

</semantics>

</math><img src="./e9d471705385cabc4294ba359b52f717dc48f5af.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:12.783ex; height:3.009ex;" alt="{\displaystyle \{(x_{i},y_{i})\}_{i=1}^{n},}" loading="lazy"> a differentiable loss function <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle L(y,F(x)),}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle L(y,F(x)),}</annotation>

</semantics>

</math><img src="./2e266db7246eb12c95f679fe2be99ffc00e12c2a.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:11.108ex; height:2.843ex;" alt="{\displaystyle L(y,F(x)),}" loading="lazy"> number of iterations M.

Algorithm:

<ol><li>Initialize model with a constant value:

<msub>

</mrow>

</msub>

<mrow>

</mrow>

</munder>

</mrow>

</mrow>

</mrow>

</munderover>

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle F_{0}(x)={\underset {\gamma }{\arg \min }}\sum _{i=1}^{n}L(y_{i},\gamma ).}</annotation>

</semantics>

</math><img src="./0f62a6e1bf376af3c3e0ac34987f4b76b53a0207.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -3.005ex; width:28.688ex; height:6.843ex;" alt="{\displaystyle F_{0}(x)={\underset {\gamma }{\arg \min }}\sum _{i=1}^{n}L(y_{i},\gamma ).}" loading="lazy"></dd></dl></li>

<li>For m = 1 to M:

<ol><li>Compute so-called pseudo-residuals:

<msub>

</mrow>

</msub>

<msub>

<mrow>

<mfrac>

<mrow>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mrow>

<mrow>

<msub>

</mrow>

</msub>

</mrow>

</mfrac>

</mrow>

</mrow>

<msub>

</mrow>

</msub>

</mrow>

</msub>

</mstyle>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle r_{im}=-\left[{\frac {\partial L(y_{i},F(x_{i}))}{\partial F(x_{i})}}\right]_{F(x)=F_{m-1}(x)}\quad {\mbox{for }}i=1,\ldots ,n.}</annotation>

</semantics>

</math><img src="./724e7996c78440be2dce585a00e39ca9c561775b.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -3.171ex; width:54.102ex; height:7.009ex;" alt="{\displaystyle r_{im}=-\left[{\frac {\partial L(y_{i},F(x_{i}))}{\partial F(x_{i})}}\right]_{F(x)=F_{m-1}(x)}\quad {\mbox{for }}i=1,\ldots ,n.}" loading="lazy"></dd></dl></li>

<li>Fit a base learner (or weak learner, e.g. tree) closed under scaling <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle h_{m}(x)}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle h_{m}(x)}</annotation>

</semantics>

</math><img src="./df5d8d5ba1497f33f67eb1721a3bb7616b32ad8e.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:6.153ex; height:2.843ex;" alt="{\displaystyle h_{m}(x)}" loading="lazy"> to pseudo-residuals, i.e. train it using the training set <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \{(x_{i},r_{im})\}_{i=1}^{n}}">

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mrow>

</mrow>

</msubsup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \{(x_{i},r_{im})\}_{i=1}^{n}}</annotation>

</semantics>

</math><img src="./04851884fee624ad5fc326047e6a0df1bfe8e28d.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:13.489ex; height:3.009ex;" alt="{\displaystyle \{(x_{i},r_{im})\}_{i=1}^{n}}" loading="lazy">.</li>

<li>Compute multiplier <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \gamma _{m}}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \gamma _{m}}</annotation>

</semantics>

</math><img src="./27a890b6f03d91a6f3f1033e4ffdf6b2c46c7737.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:2.879ex; height:2.176ex;" alt="{\displaystyle \gamma _{m}}" loading="lazy"> by solving the following one-dimensional optimization problem:

<msub>

</mrow>

</msub>

</mrow>

</munder>

</mrow>

</mrow>

</mrow>

</munderover>

<mrow>

<mrow>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mrow>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \gamma _{m}={\underset {\gamma }{\operatorname {arg\,min} }}\sum _{i=1}^{n}L\left(y_{i},F_{m-1}(x_{i})+\gamma h_{m}(x_{i})\right).}</annotation>

</semantics>

</math><img src="./bb264b744f04173741604889d90ec89f53c81119.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -3.005ex; width:45.656ex; height:6.843ex;" alt="{\displaystyle \gamma _{m}={\underset {\gamma }{\operatorname {arg\,min} }}\sum _{i=1}^{n}L\left(y_{i},F_{m-1}(x_{i})+\gamma h_{m}(x_{i})\right).}" loading="lazy"></dd></dl></li>

<li>Update the model:

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle F_{m}(x)=F_{m-1}(x)+\gamma _{m}h_{m}(x).}</annotation>

</semantics>

</math><img src="./cecd6e7dc902fad2070d1c19ba763ce7fe6440bb.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:30.336ex; height:2.843ex;" alt="{\displaystyle F_{m}(x)=F_{m-1}(x)+\gamma _{m}h_{m}(x).}" loading="lazy"></dd></dl></li></ol></li>

<li>Output <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle F_{M}(x).}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle F_{M}(x).}</annotation>

</semantics>

</math><img src="./146487545c36538312f6f6bbdad5e342f3649015.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:7.24ex; height:2.843ex;" alt="{\displaystyle F_{M}(x).}" loading="lazy"></li></ol>

</div></div>

<div class="mw-heading mw-heading2"><h2 id="Gradient_tree_boosting">Gradient tree boosting</h2></div>

Gradient boosting is typically used with <a href="Decision_tree_learning" title="Decision tree learning">decision trees</a> (especially <a href="Classification_and_regression_tree" class="mw-redirect" title="Classification and regression tree">CARTs</a>) of a fixed size as base learners. For this special case, Friedman proposes a modification to gradient boosting method which improves the quality of fit of each base learner.

Generic gradient boosting at the m-th step would fit a decision tree <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle h_{m}(x)}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle h_{m}(x)}</annotation>

</semantics>

</math><img src="./df5d8d5ba1497f33f67eb1721a3bb7616b32ad8e.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:6.153ex; height:2.843ex;" alt="{\displaystyle h_{m}(x)}" loading="lazy"> to pseudo-residuals. Let <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle J_{m}}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle J_{m}}</annotation>

</semantics>

</math><img src="./0f44de3c71c1dc3fd38046682dd264767176cb04.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:2.965ex; height:2.509ex;" alt="{\displaystyle J_{m}}" loading="lazy"> be the number of its leaves. The tree partitions the input space into <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle J_{m}}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle J_{m}}</annotation>

</semantics>

</math><img src="./0f44de3c71c1dc3fd38046682dd264767176cb04.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:2.965ex; height:2.509ex;" alt="{\displaystyle J_{m}}" loading="lazy"> disjoint regions <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle R_{1m},\ldots ,R_{J_{m}m}}">

<msub>

</mrow>

</msub>

<msub>

<msub>

</mrow>

</msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle R_{1m},\ldots ,R_{J_{m}m}}</annotation>

</semantics>

</math><img src="./f39f7108d743eb15776305f01b65f97a6c369f15.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:15.126ex; height:2.676ex;" alt="{\displaystyle R_{1m},\ldots ,R_{J_{m}m}}" loading="lazy"> and predicts a constant value in each region. Using the <a href="Indicator_notation" class="mw-redirect" title="Indicator notation">indicator notation</a>, the output of <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle h_{m}(x)}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle h_{m}(x)}</annotation>

</semantics>

</math><img src="./df5d8d5ba1497f33f67eb1721a3bb7616b32ad8e.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:6.153ex; height:2.843ex;" alt="{\displaystyle h_{m}(x)}" loading="lazy"> for input x can be written as the sum:

<msub>

</mrow>

</msub>

</mrow>

<msub>

</mrow>

</msub>

</mrow>

</munderover>

<msub>

</mrow>

</msub>

<msub>

</mrow>

<msub>

</mrow>

</msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle h_{m}(x)=\sum _{j=1}^{J_{m}}b_{jm}\mathbf {1} _{R_{jm}}(x),}</annotation>

</semantics>

</math><img src="./70cbdeb230638d61f5eef24cbc69135e064d6fb7.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -3.338ex; width:24.831ex; height:7.676ex;" alt="{\displaystyle h_{m}(x)=\sum _{j=1}^{J_{m}}b_{jm}\mathbf {1} _{R_{jm}}(x),}" loading="lazy"></dd></dl>

where <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle b_{jm}}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle b_{jm}}</annotation>

</semantics>

</math><img src="./3e39da3293168a3a5f70282326cc6b4ed7519ec9.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:3.35ex; height:2.843ex;" alt="{\displaystyle b_{jm}}" loading="lazy"> is the value predicted in the region <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle R_{jm}}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle R_{jm}}</annotation>

</semantics>

</math><img src="./39c0c8f28587e7408b7984127ec0a02c1f0aa3bb.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:4.116ex; height:2.843ex;" alt="{\displaystyle R_{jm}}" loading="lazy">.<a href="#cite_note-9">[9]</a>

Then the coefficients <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle b_{jm}}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle b_{jm}}</annotation>

</semantics>

</math><img src="./3e39da3293168a3a5f70282326cc6b4ed7519ec9.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:3.35ex; height:2.843ex;" alt="{\displaystyle b_{jm}}" loading="lazy"> are multiplied by some value <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \gamma _{m}}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \gamma _{m}}</annotation>

</semantics>

</math><img src="./27a890b6f03d91a6f3f1033e4ffdf6b2c46c7737.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:2.879ex; height:2.176ex;" alt="{\displaystyle \gamma _{m}}" loading="lazy">, chosen using line search so as to minimize the loss function, and the model is updated as follows:

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mrow>

</munder>

</mrow>

</mrow>

</mrow>

</munderover>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle F_{m}(x)=F_{m-1}(x)+\gamma _{m}h_{m}(x),\quad \gamma _{m}={\underset {\gamma }{\operatorname {arg\,min} }}\sum _{i=1}^{n}L(y_{i},F_{m-1}(x_{i})+\gamma h_{m}(x_{i})).}</annotation>

</semantics>

</math><img src="./b860dcc5aa93fa3609101acfe0657948f146253e.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -3.005ex; width:77.927ex; height:6.843ex;" alt="{\displaystyle F_{m}(x)=F_{m-1}(x)+\gamma _{m}h_{m}(x),\quad \gamma _{m}={\underset {\gamma }{\operatorname {arg\,min} }}\sum _{i=1}^{n}L(y_{i},F_{m-1}(x_{i})+\gamma h_{m}(x_{i})).}" loading="lazy"></dd></dl>

Friedman proposes to modify this algorithm so that it chooses a separate optimal value <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \gamma _{jm}}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \gamma _{jm}}</annotation>

</semantics>

</math><img src="./b496550eb93b8ad405a8466ddf461b674e6fa6c6.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:3.557ex; height:2.343ex;" alt="{\displaystyle \gamma _{jm}}" loading="lazy"> for each of the tree's regions, instead of a single <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \gamma _{m}}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \gamma _{m}}</annotation>

</semantics>

</math><img src="./27a890b6f03d91a6f3f1033e4ffdf6b2c46c7737.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:2.879ex; height:2.176ex;" alt="{\displaystyle \gamma _{m}}" loading="lazy"> for the whole tree. He calls the modified algorithm "TreeBoost". The coefficients <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle b_{jm}}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle b_{jm}}</annotation>

</semantics>

</math><img src="./3e39da3293168a3a5f70282326cc6b4ed7519ec9.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:3.35ex; height:2.843ex;" alt="{\displaystyle b_{jm}}" loading="lazy"> from the tree-fitting procedure can be then simply discarded and the model update rule becomes:

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mrow>

<msub>

</mrow>

</msub>

</mrow>

</munderover>

<msub>

</mrow>

</msub>

<msub>

</mrow>

<msub>

</mrow>

</msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mrow>

</munder>

</mrow>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mrow>

</munder>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle F_{m}(x)=F_{m-1}(x)+\sum _{j=1}^{J_{m}}\gamma _{jm}\mathbf {1} _{R_{jm}}(x),\quad \gamma _{jm}={\underset {\gamma }{\operatorname {arg\,min} }}\sum _{x_{i}\in R_{jm}}L(y_{i},F_{m-1}(x_{i})+\gamma ).}</annotation>

</semantics>

</math><img src="./32d00130b1eb1fd43116ca01dd80d4343bda624e.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -3.671ex; width:80.199ex; height:8.009ex;" alt="{\displaystyle F_{m}(x)=F_{m-1}(x)+\sum _{j=1}^{J_{m}}\gamma _{jm}\mathbf {1} _{R_{jm}}(x),\quad \gamma _{jm}={\underset {\gamma }{\operatorname {arg\,min} }}\sum _{x_{i}\in R_{jm}}L(y_{i},F_{m-1}(x_{i})+\gamma ).}" loading="lazy"></dd></dl>

When the loss <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle L(\cdot ,\cdot )}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle L(\cdot ,\cdot )}</annotation>

</semantics>

</math><img src="./94cac973f6f72b76452e35b53ed8aae57f1fa1c3.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:5.72ex; height:2.843ex;" alt="{\displaystyle L(\cdot ,\cdot )}" loading="lazy"> is mean-squared error (MSE) the coefficients <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \gamma _{jm}}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \gamma _{jm}}</annotation>

</semantics>

</math><img src="./b496550eb93b8ad405a8466ddf461b674e6fa6c6.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:3.557ex; height:2.343ex;" alt="{\displaystyle \gamma _{jm}}" loading="lazy"> coincide with the coefficients of the tree-fitting procedure <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle b_{jm}}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle b_{jm}}</annotation>

</semantics>

</math><img src="./3e39da3293168a3a5f70282326cc6b4ed7519ec9.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:3.35ex; height:2.843ex;" alt="{\displaystyle b_{jm}}" loading="lazy">.

The number <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle J}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle J}</annotation>

</semantics>

</math><img src="./359e4f407b49910e02c27c2f52e87a36cd74c053.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.471ex; height:2.176ex;" alt="{\displaystyle J}" loading="lazy"> of terminal nodes in the trees is a parameter which controls the maximum allowed level of <a href="Interaction_(statistics)" title="Interaction (statistics)">interaction</a> between variables in the model. With <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle J=2}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle J=2}</annotation>

</semantics>

</math><img src="./01a05a683da4199349749539d52eed799fc9b444.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:5.732ex; height:2.176ex;" alt="{\displaystyle J=2}" loading="lazy"> (<a href="Decision_stump" title="Decision stump">decision stumps</a>), no interaction between variables is allowed. With <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle J=3}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle J=3}</annotation>

</semantics>

</math><img src="./6a539436f8e4b82b9a15c8c568d4f5f2ed20e9d9.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:5.732ex; height:2.176ex;" alt="{\displaystyle J=3}" loading="lazy"> the model may include effects of the interaction between up to two variables, and so on. <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle J}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle J}</annotation>

</semantics>

</math><img src="./359e4f407b49910e02c27c2f52e87a36cd74c053.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.471ex; height:2.176ex;" alt="{\displaystyle J}" loading="lazy"> can be adjusted for a data set at hand.

Hastie et al.<a href="#cite_note-hastie-1">[1]</a> comment that typically <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle 4\leq J\leq 8}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle 4\leq J\leq 8}</annotation>

</semantics>

</math><img src="./f5538cb4c32e0d2fd35ac448cfd32be8536435cb.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.505ex; width:9.993ex; height:2.343ex;" alt="{\displaystyle 4\leq J\leq 8}" loading="lazy"> work well for boosting and results are fairly insensitive to the choice of <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle J}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle J}</annotation>

</semantics>

</math><img src="./359e4f407b49910e02c27c2f52e87a36cd74c053.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.471ex; height:2.176ex;" alt="{\displaystyle J}" loading="lazy"> in this range, <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle J=2}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle J=2}</annotation>

</semantics>

</math><img src="./01a05a683da4199349749539d52eed799fc9b444.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:5.732ex; height:2.176ex;" alt="{\displaystyle J=2}" loading="lazy"> is insufficient for many applications, and <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle J>10}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle J>10}</annotation>

</semantics>

</math><img src="./29ba1ea13bd5424eb0dde3348d3849124707daed.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:6.895ex; height:2.176ex;" alt="{\displaystyle J>10}" loading="lazy"> is unlikely to be required.

<div class="mw-heading mw-heading2"><h2 id="Regularization">Regularization</h2></div>

Fitting the training set too closely can lead to degradation of the model's generalization ability, that is, its performance on unseen examples. Several so-called <a href="Regularization_(mathematics)" title="Regularization (mathematics)">regularization</a> techniques reduce this <a href="Overfitting" title="Overfitting">overfitting</a> effect by constraining the fitting procedure.

One natural regularization parameter is the number of gradient boosting iterations M (i.e. the number of base models). Increasing M reduces the error on training set, but increases risk of overfitting. An optimal value of M is often selected by monitoring prediction error on a separate validation data set.

Another regularization parameter for tree boosting is tree depth. The higher this value the more likely the model will overfit the training data.

<div class="mw-heading mw-heading3"><h3 id="Shrinkage">Shrinkage</h3></div>

An important part of gradient boosting is regularization by shrinkage which uses a modified update rule:

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle F_{m}(x)=F_{m-1}(x)+\nu \cdot \gamma _{m}h_{m}(x),\quad 0<\nu \leq 1,}</annotation>

</semantics>

</math><img src="./0b737e15d88c6cb54951f3eb47faf3db0731d4e8.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:46.357ex; height:2.843ex;" alt="{\displaystyle F_{m}(x)=F_{m-1}(x)+\nu \cdot \gamma _{m}h_{m}(x),\quad 0<\nu \leq 1,}" loading="lazy"></dd></dl>

where <a href="Hyperparameter_(machine_learning)" title="Hyperparameter (machine learning)">parameter</a> <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \nu }">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \nu }</annotation>

</semantics>

</math><img src="./c15bbbb971240cf328aba572178f091684585468.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.232ex; height:1.676ex;" alt="{\displaystyle \nu }" loading="lazy"> is called the "learning rate".

Empirically, it has been found that using small <a href="Learning_rate" title="Learning rate">learning rates</a> (such as <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \nu <0.1}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \nu <0.1}</annotation>

</semantics>

</math><img src="./1dc11845c4ef971b8ee52190037c1c881bf8fcda.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:7.302ex; height:2.176ex;" alt="{\displaystyle \nu <0.1}" loading="lazy">) yields dramatic improvements in models' generalization ability over gradient boosting without shrinking (<math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \nu =1}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \nu =1}</annotation>

</semantics>

</math><img src="./c2ca7fd51cd02b2c4311446c6d6b0f53b2c529cf.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:5.493ex; height:2.176ex;" alt="{\displaystyle \nu =1}" loading="lazy">).<a href="#cite_note-hastie-1">[1]</a> However, it comes at the price of increasing <a href="Computational_time" class="mw-redirect" title="Computational time">computational time</a> both during training and <a href="Information_retrieval" title="Information retrieval">querying</a>: lower learning rate requires more iterations.

<div class="mw-heading mw-heading3"><h3 id="Stochastic_gradient_boosting">Stochastic gradient boosting</h3></div>

Soon after the introduction of gradient boosting, Friedman proposed a minor modification to the algorithm, motivated by <a href="Leo_Breiman" title="Leo Breiman">Breiman</a>'s <a href="Bootstrap_aggregation" class="mw-redirect" title="Bootstrap aggregation">bootstrap aggregation</a> ("bagging") method.<a href="#cite_note-Friedman1999b-2">[2]</a> Specifically, he proposed that at each iteration of the algorithm, a base learner should be fit on a subsample of the training set drawn at random without replacement.<a href="#cite_note-10">[10]</a> Friedman observed a substantial improvement in gradient boosting's accuracy with this modification.

Subsample size is some constant fraction <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle f}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle f}</annotation>

</semantics>

</math><img src="./132e57acb643253e7810ee9702d9581f159a1c61.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:1.279ex; height:2.509ex;" alt="{\displaystyle f}" loading="lazy"> of the size of the training set. When <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle f=1}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle f=1}</annotation>

</semantics>

</math><img src="./60eab5895ca5fe74555e9d59e499c94e1d2ef3f4.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:5.54ex; height:2.509ex;" alt="{\displaystyle f=1}" loading="lazy">, the algorithm is deterministic and identical to the one described above. Smaller values of <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle f}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle f}</annotation>

</semantics>

</math><img src="./132e57acb643253e7810ee9702d9581f159a1c61.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:1.279ex; height:2.509ex;" alt="{\displaystyle f}" loading="lazy"> introduce randomness into the algorithm and help prevent <a href="Overfitting" title="Overfitting">overfitting</a>, acting as a kind of <a href="Regularization_(mathematics)" title="Regularization (mathematics)">regularization</a>. The algorithm also becomes faster, because regression trees have to be fit to smaller datasets at each iteration. Friedman<a href="#cite_note-Friedman1999b-2">[2]</a> obtained that <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle 0.5\leq f\leq 0.8}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle 0.5\leq f\leq 0.8}</annotation>

</semantics>

</math><img src="./a327edb35a0e4d5ade75128a32f46d1c6522d166.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:13.419ex; height:2.509ex;" alt="{\displaystyle 0.5\leq f\leq 0.8}" loading="lazy"> leads to good results for small and moderate sized training sets. Therefore, <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle f}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle f}</annotation>

</semantics>

</math><img src="./132e57acb643253e7810ee9702d9581f159a1c61.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:1.279ex; height:2.509ex;" alt="{\displaystyle f}" loading="lazy"> is typically set to 0.5, meaning that one half of the training set is used to build each base learner.

Also, like in bagging, subsampling allows one to define an <a href="Out-of-bag_error" title="Out-of-bag error">out-of-bag error</a> of the prediction performance improvement by evaluating predictions on those observations which were not used in the building of the next base learner. Out-of-bag estimates help avoid the need for an independent validation dataset, but often underestimate actual performance improvement and the optimal number of iterations.<a href="#cite_note-gbm-vignette-11">[11]</a><a href="#cite_note-12">[12]</a>

<div class="mw-heading mw-heading3"><h3 id="Number_of_observations_in_leaves">Number of observations in leaves</h3></div>

Gradient tree boosting implementations often also use regularization by limiting the minimum number of observations in trees' terminal nodes. It is used in the tree building process by ignoring any splits that lead to nodes containing fewer than this number of training set instances.

Imposing this limit helps to reduce variance in predictions at leaves.

<div class="mw-heading mw-heading3"><h3 id="Complexity_penalty">Complexity penalty</h3></div>

Another useful regularization technique for gradient boosted model is to penalize its complexity.<a href="#cite_note-13">[13]</a> For gradient boosted trees, model complexity can be defined as the proportional number of leaves in the trees. The joint optimization of loss and model complexity corresponds to a post-pruning algorithm to remove branches that fail to reduce the loss by a threshold.

Other kinds of regularization such as an <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \ell _{2}}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \ell _{2}}</annotation>

</semantics>

</math><img src="./85a4571ee9be10bd3c9df2480ab3d280f99e801a.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:2.024ex; height:2.509ex;" alt="{\displaystyle \ell _{2}}" loading="lazy"> penalty on the leaf values can also be used to avoid <a href="Overfitting" title="Overfitting">overfitting</a>.

<div class="mw-heading mw-heading2"><h2 id="Usage">Usage</h2></div>

Gradient boosting can be used in the field of <a href="Learning_to_rank" title="Learning to rank">learning to rank</a>. The commercial web search engines <a href="Yahoo" title="Yahoo">Yahoo</a><a href="#cite_note-14">[14]</a> and <a href="Yandex" title="Yandex">Yandex</a><a href="#cite_note-snezhinsk-15">[15]</a> use variants of gradient boosting in their machine-learned ranking engines. Gradient boosting is also utilized in High Energy Physics in data analysis. At the Large Hadron Collider (LHC), variants of gradient boosting Deep Neural Networks (DNN) were successful in reproducing the results of non-machine learning methods of analysis on datasets used to discover the <a href="Higgs_boson" title="Higgs boson">Higgs boson</a>.<a href="#cite_note-16">[16]</a> Gradient boosting decision tree was also applied in earth and geological studies – for example quality evaluation of sandstone reservoir.<a href="#cite_note-17">[17]</a>

<div class="mw-heading mw-heading2"><h2 id="Names">Names</h2></div>

The method goes by a variety of names. Friedman introduced his regression technique as a "Gradient Boosting Machine" (GBM).<a href="#cite_note-Friedman1999a-4">[4]</a> Mason, Baxter et al. described the generalized abstract class of algorithms as "functional gradient boosting".<a href="#cite_note-MasonBaxterBartlettFrean1999a-5">[5]</a><a href="#cite_note-MasonBaxterBartlettFrean1999b-6">[6]</a> Friedman et al. describe an advancement of gradient boosted models as Multiple Additive Regression Trees (MART);<a href="#cite_note-18">[18]</a> Elith et al. describe that approach as "Boosted Regression Trees" (BRT).<a href="#cite_note-19">[19]</a>

A popular open-source implementation for <a href="R_(programming_language)" title="R (programming language)">R</a> calls it a "Generalized Boosting Model",<a href="#cite_note-gbm-vignette-11">[11]</a> however packages expanding this work use BRT.<a href="#cite_note-20">[20]</a> Yet another name is TreeNet, after an early commercial implementation from Salford System's Dan Steinberg, one of researchers who pioneered the use of tree-based methods.<a href="#cite_note-21">[21]</a>

<div class="mw-heading mw-heading2"><h2 id="Feature_importance_ranking">Feature importance ranking</h2></div>

Gradient boosting can be used for feature importance ranking, which is usually based on aggregating importance function of the base learners.<a href="#cite_note-:1-22">[22]</a> For example, if a gradient boosted trees algorithm is developed using entropy-based <a href="Decision_tree_learning" title="Decision tree learning">decision trees</a>, the ensemble algorithm ranks the importance of features based on entropy as well with the caveat that it is averaged out over all base learners.<a href="#cite_note-:1-22">[22]</a><a href="#cite_note-hastie-1">[1]</a>

<div class="mw-heading mw-heading2"><h2 id="Disadvantages">Disadvantages</h2></div>

While boosting can increase the accuracy of a base learner, such as a decision tree or linear regression, it sacrifices intelligibility and <a href="Interpretability" title="Interpretability">interpretability</a>.<a href="#cite_note-:1-22">[22]</a><a href="#cite_note-23">[23]</a> For example, following the path that a decision tree takes to make its decision is trivial and self-explained, but following the paths of hundreds or thousands of trees is much harder. To achieve both performance and interpretability, some model compression techniques allow transforming an XGBoost into a single "born-again" decision tree that approximates the same decision function.<a href="#cite_note-24">[24]</a> Furthermore, its implementation may be more difficult due to the higher computational demand.

<ul><li><a href="AdaBoost" title="AdaBoost">AdaBoost</a></li>

<li><a href="Random_forest" title="Random forest">Random forest</a></li>

<li><a href="Catboost" class="mw-redirect" title="Catboost">Catboost</a></li>

<li><a href="LightGBM" title="LightGBM">LightGBM</a></li>

<li><a href="XGBoost" title="XGBoost">XGBoost</a></li>

<li><a href="Decision_tree_learning" title="Decision tree learning">Decision tree learning</a></li></ul>

<div class="mw-heading mw-heading2"><h2 id="References">References</h2></div>

/* start https://en.wikipedia.org/ */

.mw-parser-output .reflist{margin-bottom:0.5em;list-style-type:decimal}@media screen{.mw-parser-output .reflist{font-size:90%}}.mw-parser-output .reflist .references{font-size:100%;margin-bottom:0;list-style-type:inherit}.mw-parser-output .reflist-columns-2{column-width:30em}.mw-parser-output .reflist-columns-3{column-width:25em}.mw-parser-output .reflist-columns{margin-top:0.3em}.mw-parser-output .reflist-columns ol{margin-top:0}.mw-parser-output .reflist-columns li{page-break-inside:avoid;break-inside:avoid-column}.mw-parser-output .reflist-upper-alpha{list-style-type:upper-alpha}.mw-parser-output .reflist-upper-roman{list-style-type:upper-roman}.mw-parser-output .reflist-lower-alpha{list-style-type:lower-alpha}.mw-parser-output .reflist-lower-greek{list-style-type:lower-greek}.mw-parser-output .reflist-lower-roman{list-style-type:lower-roman}

/* end https://en.wikipedia.org/ */

</style><div class="reflist">

/* start https://en.wikipedia.org/ */

.mw-parser-output cite.citation{font-style:inherit;word-wrap:break-word}.mw-parser-output .citation q{quotes:"\"""\"""'""'"}.mw-parser-output .citation:target{background-color:rgba(0,127,255,0.133)}.mw-parser-output .id-lock-free.id-lock-free a{background:url("./mw/Lock-green.svg")right 0.1em center/9px no-repeat}.mw-parser-output .id-lock-limited.id-lock-limited a,.mw-parser-output .id-lock-registration.id-lock-registration a{background:url("./mw/Lock-gray-alt-2.svg")right 0.1em center/9px no-repeat}.mw-parser-output .id-lock-subscription.id-lock-subscription a{background:url("./mw/Lock-red-alt-2.svg")right 0.1em center/9px no-repeat}.mw-parser-output .cs1-ws-icon a{background:url("./mw/Wikisource-logo.svg")right 0.1em center/12px no-repeat}body:not(.skin-timeless):not(.skin-minerva) .mw-parser-output .id-lock-free a,body:not(.skin-timeless):not(.skin-minerva) .mw-parser-output .id-lock-limited a,body:not(.skin-timeless):not(.skin-minerva) .mw-parser-output .id-lock-registration a,body:not(.skin-timeless):not(.skin-minerva) .mw-parser-output .id-lock-subscription a,body:not(.skin-timeless):not(.skin-minerva) .mw-parser-output .cs1-ws-icon a{background-size:contain;padding:0 1em 0 0}.mw-parser-output .cs1-code{color:inherit;background:inherit;border:none;padding:inherit}.mw-parser-output .cs1-hidden-error{display:none;color:var(--color-error,#d33)}.mw-parser-output .cs1-visible-error{color:var(--color-error,#d33)}.mw-parser-output .cs1-maint{display:none;color:#085;margin-left:0.3em}.mw-parser-output .cs1-kern-left{padding-left:0.2em}.mw-parser-output .cs1-kern-right{padding-right:0.2em}.mw-parser-output .citation .mw-selflink{font-weight:inherit}@media screen{.mw-parser-output .cs1-format{font-size:95%}html.skin-theme-clientpref-night .mw-parser-output .cs1-maint{color:#18911f}}@media screen and (prefers-color-scheme:dark){html.skin-theme-clientpref-os .mw-parser-output .cs1-maint{color:#18911f}}

/* end https://en.wikipedia.org/ */

</style><cite id="CITEREFHastieTibshiraniFriedman2009" class="citation book cs1">Hastie, T.; Tibshirani, R.; Friedman, J. H. (2009). <a rel="nofollow" class="external text" href="https://web.archive.org/web/20091110212529/http://www-stat.stanford.edu/~tibs/ElemStatLearn/">"10. Boosting and Additive Trees"</a>. The Elements of Statistical Learning (2nd ed.). New York: Springer. pp. 337–384. <a href="ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <bdi>978-0-387-84857-0</bdi>. Archived from <a rel="nofollow" class="external text" href="http://www-stat.stanford.edu/~tibs/ElemStatLearn/">the original</a> on 2009-11-10.</cite>

</li>

<li id="cite_note-Friedman1999b-2">^ <a href="#cite_ref-Friedman1999b_2-0">a</a> <a href="#cite_ref-Friedman1999b_2-1">b</a> <a href="#cite_ref-Friedman1999b_2-2">c</a> <a href="#cite_ref-Friedman1999b_2-3">d</a> <cite id="CITEREFFriedman1999" class="citation web cs1">Friedman, J. H. (March 1999). <a rel="nofollow" class="external text" href="https://web.archive.org/web/20140801033113/http://statweb.stanford.edu/~jhf/ftp/stobst.pdf">"Stochastic Gradient Boosting"</a> (PDF). Archived from <a rel="nofollow" class="external text" href="https://statweb.stanford.edu/~jhf/ftp/stobst.pdf">the original</a> (PDF) on 2014-08-01. Retrieved 2013-11-13.</cite>

</li>

<li id="cite_note-Breiman1997-3"><a href="#cite_ref-Breiman1997_3-0">^</a> <cite id="CITEREFBreiman1997" class="citation journal cs1">Breiman, L. (June 1997). <a rel="nofollow" class="external text" href="https://statistics.berkeley.edu/sites/default/files/tech-reports/486.pdf">"Arcing The Edge"</a> (PDF). Technical Report 486. Statistics Department, University of California, Berkeley.</cite>

</li>

<li id="cite_note-Friedman1999a-4">^ <a href="#cite_ref-Friedman1999a_4-0">a</a> <a href="#cite_ref-Friedman1999a_4-1">b</a> <a href="#cite_ref-Friedman1999a_4-2">c</a> <cite id="CITEREFFriedman1999" class="citation web cs1">Friedman, J. H. (February 1999). <a rel="nofollow" class="external text" href="https://web.archive.org/web/20191101082737/http://statweb.stanford.edu/~jhf/ftp/trebst.pdf">"Greedy Function Approximation: A Gradient Boosting Machine"</a> (PDF). Archived from <a rel="nofollow" class="external text" href="https://statweb.stanford.edu/~jhf/ftp/trebst.pdf">the original</a> (PDF) on 2019-11-01. Retrieved 2018-08-27.</cite>

</li>

<cite id="CITEREFMasonBaxterBartlettFrean1999" class="citation conference cs1">Mason, L.; Baxter, J.; Bartlett, P. L.; Frean, Marcus (1999). <a rel="nofollow" class="external text" href="http://papers.nips.cc/paper/1766-boosting-algorithms-as-gradient-descent.pdf">"Boosting Algorithms as Gradient Descent"</a> (PDF). In <a href="Sara_Solla" title="Sara Solla">S.A. Solla</a> and T.K. Leen and K. Müller (ed.). Advances in Neural Information Processing Systems 12. MIT Press. pp. 512–518.</cite>

</li>

<cite id="CITEREFMasonBaxterBartlettFrean1999" class="citation web cs1">Mason, L.; Baxter, J.; Bartlett, P. L.; Frean, Marcus (May 1999). <a rel="nofollow" class="external text" href="https://web.archive.org/web/20181222170928/https://www.maths.dur.ac.uk/~dma6kp/pdf/face_recognition/Boosting/Mason99AnyboostLong.pdf">"Boosting Algorithms as Gradient Descent in Function Space"</a> (PDF). Archived from <a rel="nofollow" class="external text" href="https://www.maths.dur.ac.uk/~dma6kp/pdf/face_recognition/Boosting/Mason99AnyboostLong.pdf">the original</a> (PDF) on 2018-12-22.</cite>

</li>

<li id="cite_note-7"><a href="#cite_ref-7">^</a> <cite id="CITEREFCheng_Li" class="citation web cs1">Cheng Li. <a rel="nofollow" class="external text" href="http://www.chengli.io/tutorials/gradient_boosting.pdf">"A Gentle Introduction to Gradient Boosting"</a> (PDF).</cite>

</li>

<li id="cite_note-8"><a href="#cite_ref-8">^</a> <cite id="CITEREFLambers2011–2012" class="citation web cs1">Lambers, Jim (2011–2012). <a rel="nofollow" class="external text" href="https://www.math.usm.edu/lambers/mat419/lecture10.pdf">"The Method of Steepest Descent"</a> (PDF).</cite>

</li>

<li id="cite_note-9"><a href="#cite_ref-9">^</a> Note: in case of usual CART trees, the trees are fitted using least-squares loss, and so the coefficient <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle b_{jm}}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle b_{jm}}</annotation>

</semantics>

</math><img src="./3e39da3293168a3a5f70282326cc6b4ed7519ec9.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:3.35ex; height:2.843ex;" alt="{\displaystyle b_{jm}}" loading="lazy"> for the region <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle R_{jm}}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle R_{jm}}</annotation>

</semantics>

</math><img src="./39c0c8f28587e7408b7984127ec0a02c1f0aa3bb.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:4.116ex; height:2.843ex;" alt="{\displaystyle R_{jm}}" loading="lazy"> is equal to just the value of output variable, averaged over all training instances in <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle R_{jm}}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle R_{jm}}</annotation>

</semantics>

</math><img src="./39c0c8f28587e7408b7984127ec0a02c1f0aa3bb.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:4.116ex; height:2.843ex;" alt="{\displaystyle R_{jm}}" loading="lazy">.

</li>

<li id="cite_note-10"><a href="#cite_ref-10">^</a> Note that this is different from bagging, which samples with replacement because it uses samples of the same size as the training set.

</li>

<li id="cite_note-gbm-vignette-11">^ <a href="#cite_ref-gbm-vignette_11-0">a</a> <a href="#cite_ref-gbm-vignette_11-1">b</a> Ridgeway, Greg (2007). <a rel="nofollow" class="external text" href="https://cran.r-project.org/web/packages/gbm/gbm.pdf">Generalized Boosted Models: A guide to the gbm package.</a>

</li>

<li id="cite_note-12"><a href="#cite_ref-12">^</a> <a rel="nofollow" class="external text" href="https://www.analyticsvidhya.com/blog/2015/09/complete-guide-boosting-methods/">Learn Gradient Boosting Algorithm for better predictions (with codes in R)</a>

</li>

<li id="cite_note-13"><a href="#cite_ref-13">^</a> Tianqi Chen. <a rel="nofollow" class="external text" href="https://web.njit.edu/~usman/courses/cs675_fall16/BoostedTree.pdf">Introduction to Boosted Trees</a>

</li>

<li id="cite_note-14"><a href="#cite_ref-14">^</a> Cossock, David and Zhang, Tong (2008). <a rel="nofollow" class="external text" href="http://www.stat.rutgers.edu/~tzhang/papers/it08-ranking.pdf">Statistical Analysis of Bayes Optimal Subset Ranking</a> <a rel="nofollow" class="external text" href="https://web.archive.org/web/20100807162855/http://www.stat.rutgers.edu/~tzhang/papers/it08-ranking.pdf">Archived</a> 2010-08-07 at the <a href="Wayback_Machine" title="Wayback Machine">Wayback Machine</a>, page 14.

</li>

<li id="cite_note-snezhinsk-15"><a href="#cite_ref-snezhinsk_15-0">^</a> <a rel="nofollow" class="external text" href="http://webmaster.ya.ru/replies.xml?item_no=5707&ncrnd=5118">Yandex corporate blog entry about new ranking model "Snezhinsk"</a> <a rel="nofollow" class="external text" href="https://web.archive.org/web/20120301165959/http://webmaster.ya.ru/replies.xml?item_no=5707&ncrnd=5118">Archived</a> 2012-03-01 at the <a href="Wayback_Machine" title="Wayback Machine">Wayback Machine</a> (in Russian)

</li>

<li id="cite_note-16"><a href="#cite_ref-16">^</a> <cite id="CITEREFLalchand2020" class="citation arxiv cs1">Lalchand, Vidhi (2020). "Extracting more from boosted decision trees: A high energy physics case study". <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/2001.06033">2001.06033</a> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/stat.ML">stat.ML</a>].</cite>

</li>

<li id="cite_note-17"><a href="#cite_ref-17">^</a> <cite id="CITEREFMaXiaoTaoZheng2022" class="citation journal cs1">Ma, Longfei; Xiao, Hanmin; Tao, Jingwei; Zheng, Taiyi; Zhang, Haiqin (1 January 2022). <a rel="nofollow" class="external text" href="https://doi.org/10.1515%2Fgeo-2022-0354">"An intelligent approach for reservoir quality evaluation in tight sandstone reservoir using gradient boosting decision tree algorithm"</a>. Open Geosciences. 14 (1): 629–645. <a href="Bibcode_(identifier)" class="mw-redirect" title="Bibcode (identifier)">Bibcode</a>:<a rel="nofollow" class="external text" href="https://ui.adsabs.harvard.edu/abs/2022OGeo...14..354M">2022OGeo...14..354M</a>. <a href="Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1515%2Fgeo-2022-0354">10.1515/geo-2022-0354</a>. <a href="ISSN_(identifier)" class="mw-redirect" title="ISSN (identifier)">ISSN</a> <a rel="nofollow" class="external text" href="https://search.worldcat.org/issn/2391-5447">2391-5447</a>.</cite>

</li>

<li id="cite_note-18"><a href="#cite_ref-18">^</a> <cite id="CITEREFFriedman2003" class="citation journal cs1">Friedman, Jerome (2003). "Multiple Additive Regression Trees with Application in Epidemiology". Statistics in Medicine. 22 (9): 1365–1381. <a href="Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1002%2Fsim.1501">10.1002/sim.1501</a>. <a href="PMID_(identifier)" class="mw-redirect" title="PMID (identifier)">PMID</a> <a rel="nofollow" class="external text" href="https://pubmed.ncbi.nlm.nih.gov/12704603">12704603</a>. <a href="S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:41965832">41965832</a>.</cite>

</li>

<li id="cite_note-19"><a href="#cite_ref-19">^</a> <cite id="Elith2008" class="citation journal cs1">Elith, Jane (2008). <a rel="nofollow" class="external text" href="https://doi.org/10.1111%2Fj.1365-2656.2008.01390.x">"A working guide to boosted regression trees"</a>. Journal of Animal Ecology. 77 (4): 802–813. <a href="Bibcode_(identifier)" class="mw-redirect" title="Bibcode (identifier)">Bibcode</a>:<a rel="nofollow" class="external text" href="https://ui.adsabs.harvard.edu/abs/2008JAnEc..77..802E">2008JAnEc..77..802E</a>. <a href="Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1111%2Fj.1365-2656.2008.01390.x">10.1111/j.1365-2656.2008.01390.x</a>. <a href="PMID_(identifier)" class="mw-redirect" title="PMID (identifier)">PMID</a> <a rel="nofollow" class="external text" href="https://pubmed.ncbi.nlm.nih.gov/18397250">18397250</a>.</cite>

</li>

<li id="cite_note-20"><a href="#cite_ref-20">^</a> <cite id="CITEREFElith" class="citation web cs1">Elith, Jane. <a rel="nofollow" class="external text" href="https://web.archive.org/web/20200725061155/https://cran.r-project.org/web/packages/dismo/vignettes/brt.pdf">"Boosted Regression Trees for ecological modeling"</a> (PDF). CRAN. Archived from <a rel="nofollow" class="external text" href="https://cran.r-project.org/web/packages/dismo/vignettes/brt.pdf">the original</a> (PDF) on 25 July 2020. Retrieved 31 August 2018.</cite>

</li>

<li id="cite_note-21"><a href="#cite_ref-21">^</a> <cite class="citation web cs1"><a rel="nofollow" class="external text" href="https://www.kdnuggets.com/2013/06/exclusive-interview-dan-steinberg-salford-systems-data-mining-solutions-provider.html">"Exclusive: Interview with Dan Steinberg, President of Salford Systems, Data Mining Pioneer"</a>. KDnuggets.</cite>

</li>

<li id="cite_note-:1-22">^ <a href="#cite_ref-:1_22-0">a</a> <a href="#cite_ref-:1_22-1">b</a> <a href="#cite_ref-:1_22-2">c</a> <cite id="CITEREFPiryonesiEl-Diraby2020" class="citation journal cs1">Piryonesi, S. Madeh; El-Diraby, Tamer E. (2020-03-01). <a rel="nofollow" class="external text" href="https://ascelibrary.org/doi/abs/10.1061/%28ASCE%29IS.1943-555X.0000512">"Data Analytics in Asset Management: Cost-Effective Prediction of the Pavement Condition Index"</a>. Journal of Infrastructure Systems. 26 (1): 04019036. <a href="Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1061%2F%28ASCE%29IS.1943-555X.0000512">10.1061/(ASCE)IS.1943-555X.0000512</a>. <a href="ISSN_(identifier)" class="mw-redirect" title="ISSN (identifier)">ISSN</a> <a rel="nofollow" class="external text" href="https://search.worldcat.org/issn/1943-555X">1943-555X</a>. <a href="S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:213782055">213782055</a>.</cite>

</li>

<li id="cite_note-23"><a href="#cite_ref-23">^</a> <cite id="CITEREFWuKumarRoss_QuinlanGhosh2008" class="citation journal cs1">Wu, Xindong; Kumar, Vipin; Ross Quinlan, J.; Ghosh, Joydeep; Yang, Qiang; Motoda, Hiroshi; McLachlan, Geoffrey J.; Ng, Angus; Liu, Bing; Yu, Philip S.; Zhou, Zhi-Hua (2008-01-01). "Top 10 algorithms in data mining". Knowledge and Information Systems. 14 (1): 1–37. <a href="Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1007%2Fs10115-007-0114-2">10.1007/s10115-007-0114-2</a>. <a href="Hdl_(identifier)" class="mw-redirect" title="Hdl (identifier)">hdl</a>:<a rel="nofollow" class="external text" href="https://hdl.handle.net/10983%2F15329">10983/15329</a>. <a href="ISSN_(identifier)" class="mw-redirect" title="ISSN (identifier)">ISSN</a> <a rel="nofollow" class="external text" href="https://search.worldcat.org/issn/0219-3116">0219-3116</a>. <a href="S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:2367747">2367747</a>.</cite>

</li>

<li id="cite_note-24"><a href="#cite_ref-24">^</a> <cite id="CITEREFSagiRokach2021" class="citation journal cs1">Sagi, Omer; Rokach, Lior (2021). "Approximating XGBoost with an interpretable decision tree". Information Sciences. 572 (2021): 522–542. <a href="Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1016%2Fj.ins.2021.05.055">10.1016/j.ins.2021.05.055</a>.</cite>

</li>

</ol></div></div>

<div class="mw-heading mw-heading2"><h2 id="Further_reading">Further reading</h2></div>

<ul><li><cite id="CITEREFBoehmkeGreenwell2019" class="citation book cs1">Boehmke, Bradley; Greenwell, Brandon (2019). "Gradient Boosting". Hands-On Machine Learning with R. Chapman & Hall. pp. 221–245. <a href="ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <bdi>978-1-138-49568-5</bdi>.</cite></li></ul>

<div class="mw-heading mw-heading2"><h2 id="External_links">External links</h2></div>

<ul><li><a rel="nofollow" class="external text" href="http://explained.ai/gradient-boosting/index.html">How to explain gradient boosting</a></li>

<li><a rel="nofollow" class="external text" href="https://blog.datarobot.com/gradient-boosted-regression-trees">Gradient Boosted Regression Trees</a></li>

<li><a rel="nofollow" class="external text" href="https://github.com/microsoft/LightGBM">LightGBM</a></li></ul></div><div><div class="zim-footer">

This article is issued from <a class="external text" title="Last edited on 2025-06-19" href="https://en.wikipedia.org/wiki/?title=Gradient_boosting&oldid=1296429817">Wikipedia</a>. The text is available under <a class="external text" href="https://creativecommons.org/licenses/by-sa/4.0/deed.en">Creative Commons Attribution-Share Alike 4.0</a> unless otherwise noted. Additional terms may apply for the media files.

</div>

</div></div>

</div>

</main>

</div>

</body></html>